From f510e000d7f99ff24725517767fc2b14324c9fec Mon Sep 17 00:00:00 2001 From: Aman Tahiliani Date: Fri, 26 Jan 2024 19:31:47 -0500 Subject: [PATCH] Updated Output code and Readme --- MultiThreaded.py | 107 ++++++++++++++++++++++++++++++++++++++++------ README.md | 37 +++++++++++++++- SingleThreaded.py | 12 ++++-- amantahiliani.sh | 6 +++ 4 files changed, 144 insertions(+), 18 deletions(-) create mode 100755 amantahiliani.sh diff --git a/MultiThreaded.py b/MultiThreaded.py index b66f562..f2eef13 100644 --- a/MultiThreaded.py +++ b/MultiThreaded.py @@ -1,10 +1,15 @@ -__author__= "Aman Tahiliani" +__author__ = "Aman Tahiliani" import requests from bs4 import BeautifulSoup as bs from urllib.parse import urljoin import argparse import multiprocessing +import time +import matplotlib.pyplot as plt +import pandas as pd +import json + class WebCrawler: def __init__(self, seed_url): @@ -13,10 +18,9 @@ class WebCrawler: self.url_queue = manager.list() self.url_queue.append(seed_url) self.visited_urls = manager.list() - self.counter = manager.Value('i', 0) + self.counter = manager.Value("i", 0) self.h1_word_frequency = manager.dict() - # Using manager to create locks self.url_queue_lock = manager.Lock() self.visited_lock = manager.Lock() self.freq_lock = manager.Lock() @@ -42,7 +46,10 @@ class WebCrawler: with self.url_queue_lock: self.url_queue.append(urljoin(url, page_link["href"])) - filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer", "nav"])) + filtered_elements = soup.find_all( + lambda tag: tag.name in ["h1", "h2", "h3"] + and not tag.find_parents(["header", "footer", "nav"]) + ) for tags in filtered_elements: current_sentence = tags.text @@ -50,16 +57,16 @@ class WebCrawler: current_word = current_word.lower() with self.freq_lock: - current_word_dict = self.h1_word_frequency.get(current_word, {url: 0}) + current_word_dict = self.h1_word_frequency.get( + current_word, {url: 0} + ) current_word_dict[url] = current_word_dict.get(url, 0) + 1 self.h1_word_frequency[current_word] = current_word_dict return True except Exception as e: - print( - f"Exception occurred while crawling page {url}. Exception -> {e}" - ) + print(f"Exception occurred while crawling page {url}. Exception -> {e}") return False def process_url(self, new_url): @@ -74,27 +81,101 @@ class WebCrawler: visited = self.extract_page_info(new_url) def crawler(self, pages_to_parse): + crawl_start_time = time.time() + pages_per_second_list = [] + total_pages_processed = 0 with multiprocessing.Pool() as pool: while True: with self.url_queue_lock: if not self.url_queue: - break + break chunk_size = min(len(self.url_queue), multiprocessing.cpu_count()) chunks = [self.url_queue.pop(0) for chunk in range(chunk_size)] + batch_start_time = time.time() pool.starmap(self.process_url, [(url,) for url in chunks]) + batch_end_time = time.time() + + pages_processed = len(chunks) + total_pages_processed += pages_processed + time_taken = batch_end_time - batch_start_time + pages_per_second = pages_processed / time_taken + + for _ in range(pages_processed): + pages_per_second_list.append(pages_per_second) with self.visited_lock: counter_val = self.counter.value if counter_val >= pages_to_parse: break + crawl_end_time = time.time() + total_crawl_time = crawl_end_time - crawl_start_time + total_pages_crawled = self.counter.value + pages_per_minute = total_pages_crawled / (total_crawl_time / 60) + print("Done Parsing Pages") print(f"Pages Parsed {self.counter.value}/{pages_to_parse}") - print(f'Links left to parse {len(self.url_queue)}') - print(f'Words found in h1 {len(self.h1_word_frequency)}') + print(f"Links left to parse {len(self.url_queue)}") + print(f"Words found in h1 {len(self.h1_word_frequency)}") + + with open("Keywords_Output.json", "w") as json_file: + json.dump(dict(self.h1_word_frequency), json_file, indent=4) + + print( + f"Number of pages crawled vs left to be crawled -> {self.counter.value}/{len(self.url_queue)} = {self.counter.value / len(self.url_queue)} " + ) + + plt.plot(range(1, total_pages_processed + 1), pages_per_second_list) + plt.xlabel("Pages") + plt.ylabel("Pages per Second") + plt.title("Pages per Second for each page") + plt.savefig("pages_per_second.png") + + speed_table = pd.DataFrame( + { + "Total Pages Crawled": [total_pages_crawled], + "Total Crawl Time (seconds)": [total_crawl_time], + "Pages per Minute": [pages_per_minute], + } + ) + print("\nCrawl Speed in terms of Pages per Minute:") + print(speed_table) + + fig, ax = plt.subplots(figsize=(8, 4)) + ax.axis("tight") + ax.axis("off") + ax.table( + cellText=speed_table.values, + colLabels=speed_table.columns, + cellLoc="center", + loc="center", + ) + plt.savefig("crawlspeed.png") + + crawl_ratio_table = pd.DataFrame( + { + "Total Pages Crawled": [total_pages_crawled], + "Pages Left to Crawl": [len(self.url_queue)], + "Crawl Ratio": [{self.counter.value / len(self.url_queue)}], + } + ) + + fig, ax = plt.subplots(figsize=(8, 4)) + ax.axis("tight") + ax.axis("off") + ax.table( + cellText=crawl_ratio_table.values, + colLabels=crawl_ratio_table.columns, + cellLoc="center", + loc="center", + ) + plt.savefig("crawl_ratio_table.png") + if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Crawler to crawl forward from a seed URL") + parser = argparse.ArgumentParser( + description="Crawler to crawl forward from a seed URL" + ) parser.add_argument("seed_url", help="URL to start the crawling from") parser.add_argument( "--pages_to_parse", @@ -108,4 +189,4 @@ if __name__ == "__main__": web_crawler = WebCrawler(seed_url) print(f"Seed Url: {seed_url} \nNumber of Pages to Parse {pages_to_parse}") print("Initiating Crawler....") - web_crawler.crawler(pages_to_parse) \ No newline at end of file + web_crawler.crawler(pages_to_parse) diff --git a/README.md b/README.md index 3ca6609..5076f29 100644 --- a/README.md +++ b/README.md @@ -1 +1,36 @@ -# webcrawler \ No newline at end of file +# webcrawler + +## Instructions to run +1. Clone the repository +``` +git clone https://github.com/AmanTahiliani/webcrawler.git +``` +2. cd into the directory +``` +cd webcrawler +``` +3. Install the requirements +``` +pip install -r requirements.txt +``` +4. Run the shell script +``` +./amantahiliani.sh +``` + +Note: If you get a permission denied error, run the following command +``` +chmod +x amantahiliani.sh +``` +If it still doesn't work, run the following command +``` +python3 MultiThreadedCrawler.py {seed_url} --pages_to_parse=1000 +``` + + +## Output +The output is generated in the form of the following files: +1. pages_per_second.png - A graph showing the number of pages crawled per second +2. craw_ratio_table.png- A table showing the ratio of pages crawled to pages discovered +3. crawlspeed.png- A graph showing the crawl speed in pages per minute +4. Keywords_Output.json- A json file containing the keywords and the urls they were found in along with the frequency of the keyword in the url \ No newline at end of file diff --git a/SingleThreaded.py b/SingleThreaded.py index 02b6e71..64b9a9c 100644 --- a/SingleThreaded.py +++ b/SingleThreaded.py @@ -34,15 +34,19 @@ class WebCrawler: for page_link in all_page_links: self.url_queue.append(urljoin(url, page_link["href"])) - filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer","nav"])) - + filtered_elements = soup.find_all( + lambda tag: tag.name in ["h1", "h2", "h3"] + and not tag.find_parents(["header", "footer", "nav"]) + ) for tags in filtered_elements: current_sentence = tags.text for current_word in current_sentence.split(): current_word = current_word.lower() - current_word_dict = self.h1_word_frequency.get(current_word, {url:0}) + current_word_dict = self.h1_word_frequency.get( + current_word, {url: 0} + ) current_word_dict[url] = current_word_dict.get(url, 0) + 1 self.h1_word_frequency[current_word] = current_word_dict @@ -83,7 +87,7 @@ class WebCrawler: print("Done Parsing Pages") print(f"Pages Parsed {len(self.visited_urls)}/{pages_to_parse}") - print(f'Words found in h1 {len(self.h1_word_frequency)}') + print(f"Words found in h1 {len(self.h1_word_frequency)}") print(self.h1_word_frequency) # print(f'Words found in h2 {len(self.h2_word_frequency)}') # print(self.h2_word_frequency) diff --git a/amantahiliani.sh b/amantahiliani.sh new file mode 100755 index 0000000..6348aa4 --- /dev/null +++ b/amantahiliani.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +url="https://www.cc.gatech.edu" +pages_to_parse=1000 + +python3 MultiThreaded.py "$url" --pages_to_parse="$pages_to_parse" \ No newline at end of file