Updated Output code and Readme

MultiThreaded.py

@@ -5,6 +5,11 @@ from bs4 import BeautifulSoup as bs
 from urllib.parse import urljoin
 import argparse
 import multiprocessing
+import time
+import matplotlib.pyplot as plt
+import pandas as pd
+import json
+
 
 class WebCrawler:
     def __init__(self, seed_url):
@@ -13,10 +18,9 @@ class WebCrawler:
         self.url_queue = manager.list()
         self.url_queue.append(seed_url)
         self.visited_urls = manager.list()
-        self.counter = manager.Value('i', 0)
+        self.counter = manager.Value("i", 0)
         self.h1_word_frequency = manager.dict()
 
-        # Using manager to create locks
         self.url_queue_lock = manager.Lock()
         self.visited_lock = manager.Lock()
         self.freq_lock = manager.Lock()
@@ -42,7 +46,10 @@ class WebCrawler:
                 with self.url_queue_lock:
                     self.url_queue.append(urljoin(url, page_link["href"]))
 
-            filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer", "nav"]))
+            filtered_elements = soup.find_all(
+                lambda tag: tag.name in ["h1", "h2", "h3"]
+                and not tag.find_parents(["header", "footer", "nav"])
+            )
 
             for tags in filtered_elements:
                 current_sentence = tags.text
@@ -50,16 +57,16 @@ class WebCrawler:
                     current_word = current_word.lower()
 
                     with self.freq_lock:
-                        current_word_dict = self.h1_word_frequency.get(current_word, {url: 0})
+                        current_word_dict = self.h1_word_frequency.get(
+                            current_word, {url: 0}
+                        )
                         current_word_dict[url] = current_word_dict.get(url, 0) + 1
                         self.h1_word_frequency[current_word] = current_word_dict
 
             return True
 
         except Exception as e:
-            print(
+            print(f"Exception occurred while crawling page {url}. Exception -> {e}")
-                f"Exception occurred while crawling page {url}. Exception -> {e}"
-            )
             return False
 
     def process_url(self, new_url):
@@ -74,6 +81,9 @@ class WebCrawler:
             visited = self.extract_page_info(new_url)
 
     def crawler(self, pages_to_parse):
+        crawl_start_time = time.time()
+        pages_per_second_list = []
+        total_pages_processed = 0
         with multiprocessing.Pool() as pool:
             while True:
                 with self.url_queue_lock:
@@ -81,20 +91,91 @@ class WebCrawler:
                         break
                     chunk_size = min(len(self.url_queue), multiprocessing.cpu_count())
                     chunks = [self.url_queue.pop(0) for chunk in range(chunk_size)]
+                batch_start_time = time.time()
                 pool.starmap(self.process_url, [(url,) for url in chunks])
+                batch_end_time = time.time()
+
+                pages_processed = len(chunks)
+                total_pages_processed += pages_processed
+                time_taken = batch_end_time - batch_start_time
+                pages_per_second = pages_processed / time_taken
+
+                for _ in range(pages_processed):
+                    pages_per_second_list.append(pages_per_second)
 
                 with self.visited_lock:
                     counter_val = self.counter.value
                     if counter_val >= pages_to_parse:
                         break
 
+        crawl_end_time = time.time()
+        total_crawl_time = crawl_end_time - crawl_start_time
+        total_pages_crawled = self.counter.value
+        pages_per_minute = total_pages_crawled / (total_crawl_time / 60)
+
         print("Done Parsing Pages")
         print(f"Pages Parsed {self.counter.value}/{pages_to_parse}")
-        print(f'Links left to parse {len(self.url_queue)}')
+        print(f"Links left to parse {len(self.url_queue)}")
-        print(f'Words found in h1 {len(self.h1_word_frequency)}')
+        print(f"Words found in h1 {len(self.h1_word_frequency)}")
+
+        with open("Keywords_Output.json", "w") as json_file:
+            json.dump(dict(self.h1_word_frequency), json_file, indent=4)
+
+        print(
+            f"Number of pages crawled vs left to be crawled -> {self.counter.value}/{len(self.url_queue)} = {self.counter.value / len(self.url_queue)} "
+        )
+
+        plt.plot(range(1, total_pages_processed + 1), pages_per_second_list)
+        plt.xlabel("Pages")
+        plt.ylabel("Pages per Second")
+        plt.title("Pages per Second for each page")
+        plt.savefig("pages_per_second.png")
+
+        speed_table = pd.DataFrame(
+            {
+                "Total Pages Crawled": [total_pages_crawled],
+                "Total Crawl Time (seconds)": [total_crawl_time],
+                "Pages per Minute": [pages_per_minute],
+            }
+        )
+        print("\nCrawl Speed in terms of Pages per Minute:")
+        print(speed_table)
+
+        fig, ax = plt.subplots(figsize=(8, 4))
+        ax.axis("tight")
+        ax.axis("off")
+        ax.table(
+            cellText=speed_table.values,
+            colLabels=speed_table.columns,
+            cellLoc="center",
+            loc="center",
+        )
+        plt.savefig("crawlspeed.png")
+
+        crawl_ratio_table = pd.DataFrame(
+            {
+                "Total Pages Crawled": [total_pages_crawled],
+                "Pages Left to Crawl": [len(self.url_queue)],
+                "Crawl Ratio": [{self.counter.value / len(self.url_queue)}],
+            }
+        )
+
+        fig, ax = plt.subplots(figsize=(8, 4))
+        ax.axis("tight")
+        ax.axis("off")
+        ax.table(
+            cellText=crawl_ratio_table.values,
+            colLabels=crawl_ratio_table.columns,
+            cellLoc="center",
+            loc="center",
+        )
+        plt.savefig("crawl_ratio_table.png")
+
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Crawler to crawl forward from a seed URL")
+    parser = argparse.ArgumentParser(
+        description="Crawler to crawl forward from a seed URL"
+    )
     parser.add_argument("seed_url", help="URL to start the crawling from")
     parser.add_argument(
         "--pages_to_parse",

README.md

@@ -1 +1,36 @@
 # webcrawler
+
+## Instructions to run 
+1. Clone the repository
+```
+git clone https://github.com/AmanTahiliani/webcrawler.git
+```
+2. cd into the directory
+```
+cd webcrawler
+```
+3. Install the requirements
+```
+pip install -r requirements.txt
+```
+4. Run the shell script
+```
+./amantahiliani.sh 
+```
+
+Note: If you get a permission denied error, run the following command
+```
+chmod +x amantahiliani.sh
+```
+If it still doesn't work, run the following command
+```
+python3 MultiThreadedCrawler.py {seed_url} --pages_to_parse=1000
+```
+
+
+## Output
+The output is generated in the form of the following files:
+1. pages_per_second.png - A graph showing the number of pages crawled per second
+2. craw_ratio_table.png-   A table showing the ratio of pages crawled to pages discovered
+3. crawlspeed.png- A graph showing the crawl speed in pages per minute
+4. Keywords_Output.json- A json file containing the keywords and the urls they were found in along with the frequency of the keyword in the url

SingleThreaded.py

@@ -34,15 +34,19 @@ class WebCrawler:
             for page_link in all_page_links:
                 self.url_queue.append(urljoin(url, page_link["href"]))
 
-            filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer","nav"]))
+            filtered_elements = soup.find_all(
-
+                lambda tag: tag.name in ["h1", "h2", "h3"]
+                and not tag.find_parents(["header", "footer", "nav"])
+            )
 
             for tags in filtered_elements:
                 current_sentence = tags.text
                 for current_word in current_sentence.split():
                     current_word = current_word.lower()
 
-                    current_word_dict = self.h1_word_frequency.get(current_word, {url:0})
+                    current_word_dict = self.h1_word_frequency.get(
+                        current_word, {url: 0}
+                    )
                     current_word_dict[url] = current_word_dict.get(url, 0) + 1
                     self.h1_word_frequency[current_word] = current_word_dict
 
@@ -83,7 +87,7 @@ class WebCrawler:
 
         print("Done Parsing Pages")
         print(f"Pages Parsed {len(self.visited_urls)}/{pages_to_parse}")
-        print(f'Words found in h1 {len(self.h1_word_frequency)}')
+        print(f"Words found in h1 {len(self.h1_word_frequency)}")
         print(self.h1_word_frequency)
         # print(f'Words found in h2 {len(self.h2_word_frequency)}')
         # print(self.h2_word_frequency)

amantahiliani.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+url="https://www.cc.gatech.edu"
+pages_to_parse=1000
+
+python3 MultiThreaded.py "$url" --pages_to_parse="$pages_to_parse"