From the class Code as a Liberal Art · Spring 2022

Web Crawler Map

Unit 2, Tutorial 2

scraper.py

import requests
from bs4 import BeautifulSoup
import urllib.parse
import networkx as nx
import matplotlib.pyplot as plt

pages = {}

url_queue = []

url_queue.append("https://linustechtips.com/")

page_count = 0
while len(url_queue) > 0 and page_count < 50:

    next_url = url_queue.pop(0)
    while "mailto" in next_url:
        next_url = url_queue.pop(0)

    response = requests.get(next_url)

    soup = BeautifulSoup(response.text, "html.parser")

    # Process webpage here using BeautifulSoup
    print(page_count)
    print("URL: " + next_url)
    print("Title: " + soup.title.string)
    print("Length: " + str( len(soup.get_text()) ) )
    print("Contains 'backpack': " + str( "backpack" in soup.get_text().lower() ) )
    print()

    a_tags = soup.find_all("a")

    linked_pages = []
    for a in a_tags:
        url = a.get('href')
        url = urllib.parse.urljoin(next_url,url)
        if url not in url_queue:
            url_queue.append(url)
            linked_pages.append(url)

    page = {
        "title": soup.title.string,
        "length": len(soup.get_text()),
        "mentions_backpack": "backpack" in soup.get_text().lower(),
    "linked_pages": linked_pages
    }
    pages[next_url] = page

    page_count = page_count + 1

# Visualize
print("Visualizing now")

G = nx.Graph()

for key in pages:
    for link in pages[key]["linked_pages"]:
        G.add_edge(key,link)

nx.draw(G, with_labels=False, font_weight='bold')
plt.show()

Unit 2, homework 2

scraper.py

import requests
from bs4 import BeautifulSoup
import urllib.parse
import networkx as nx
import matplotlib.pyplot as plt

pages = {}

url_queue = []

url_queue.append("https://linustechtips.com/")

page_count = 0
while len(url_queue) > 0 and page_count < 50:

    next_url = url_queue.pop(0)
    while "mailto" in next_url:
        next_url = url_queue.pop(0)

    response = requests.get(next_url)

    soup = BeautifulSoup(response.text, "html.parser")

    # Process webpage here using BeautifulSoup
    print(page_count)
    print("URL: " + next_url)
    print("Title: " + soup.title.string)
    print("Length: " + str( len(soup.get_text()) ) )
    print("Contains 'backpack': " + str( "backpack" in soup.get_text().lower() ) )
    print()

    a_tags = soup.find_all("a")

    linked_pages = []
    for a in a_tags:
        url = a.get('href')
        url = urllib.parse.urljoin(next_url,url)
        if url not in url_queue:
            url_queue.append(url)
            linked_pages.append(url)

    page = {
        "title": soup.title.string,
        "length": len(soup.get_text()),
        "mentions_backpack": "backpack" in soup.get_text().lower(),
    "linked_pages": linked_pages
    }
    pages[next_url] = page

    page_count = page_count + 1

# Visualize
print("Visualizing now")

G = nx.Graph()

for key in pages:
    for link in pages[key]["linked_pages"]:
        G.add_edge(key,link)

nx.draw(G, with_labels=False, font_weight='bold')
plt.show()