amantahiliani/webcrawler
1
0
Fork 0
mirror of https://github.com/AmanTahiliani/webcrawler.git synced 2026-08-07 11:55:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

Updated Output code and Readme

Browse Source
This commit is contained in:
Aman Tahiliani
2024-01-26 19:31:47 -05:00
parent 129f0f7542
commit f510e000d7
4 changed files with 144 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
MultiThreaded.py
View File

@@ -1,10 +1,15 @@
__author__= "Aman Tahiliani"
__author__ = "Aman Tahiliani"
import requests
from bs4 import BeautifulSoup as bs
from urllib.parse import urljoin
import argparse
import multiprocessing
import time
import matplotlib.pyplot as plt
import pandas as pd
import json
class WebCrawler:
def __init__(self, seed_url):
@@ -13,10 +18,9 @@ class WebCrawler:
self.url_queue = manager.list()
self.url_queue.append(seed_url)
self.visited_urls = manager.list()
self.counter = manager.Value('i', 0)
self.counter = manager.Value("i", 0)
self.h1_word_frequency = manager.dict()
# Using manager to create locks
self.url_queue_lock = manager.Lock()
self.visited_lock = manager.Lock()
self.freq_lock = manager.Lock()
@@ -42,7 +46,10 @@ class WebCrawler:
with self.url_queue_lock:
self.url_queue.append(urljoin(url, page_link["href"]))
filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer", "nav"]))
filtered_elements = soup.find_all(
lambda tag: tag.name in ["h1", "h2", "h3"]
and not tag.find_parents(["header", "footer", "nav"])
)
for tags in filtered_elements:
current_sentence = tags.text
@@ -50,16 +57,16 @@ class WebCrawler:
current_word = current_word.lower()
with self.freq_lock:
current_word_dict = self.h1_word_frequency.get(current_word, {url: 0})
current_word_dict = self.h1_word_frequency.get(
current_word, {url: 0}
)
current_word_dict[url] = current_word_dict.get(url, 0) + 1
self.h1_word_frequency[current_word] = current_word_dict
return True
except Exception as e:
print(
f"Exception occurred while crawling page {url}. Exception -> {e}"
)
print(f"Exception occurred while crawling page {url}. Exception -> {e}")
return False
def process_url(self, new_url):
@@ -74,6 +81,9 @@ class WebCrawler:
visited = self.extract_page_info(new_url)
def crawler(self, pages_to_parse):
crawl_start_time = time.time()
pages_per_second_list = []
total_pages_processed = 0
with multiprocessing.Pool() as pool:
while True:
with self.url_queue_lock:
@@ -81,20 +91,91 @@ class WebCrawler:
break
chunk_size = min(len(self.url_queue), multiprocessing.cpu_count())
chunks = [self.url_queue.pop(0) for chunk in range(chunk_size)]
batch_start_time = time.time()
pool.starmap(self.process_url, [(url,) for url in chunks])
batch_end_time = time.time()
pages_processed = len(chunks)
total_pages_processed += pages_processed
time_taken = batch_end_time - batch_start_time
pages_per_second = pages_processed / time_taken
for _ in range(pages_processed):
pages_per_second_list.append(pages_per_second)
with self.visited_lock:
counter_val = self.counter.value
if counter_val >= pages_to_parse:
break
crawl_end_time = time.time()
total_crawl_time = crawl_end_time - crawl_start_time
total_pages_crawled = self.counter.value
pages_per_minute = total_pages_crawled / (total_crawl_time / 60)
print("Done Parsing Pages")
print(f"Pages Parsed {self.counter.value}/{pages_to_parse}")
print(f'Links left to parse {len(self.url_queue)}')
print(f'Words found in h1 {len(self.h1_word_frequency)}')
print(f"Links left to parse {len(self.url_queue)}")
print(f"Words found in h1 {len(self.h1_word_frequency)}")
with open("Keywords_Output.json", "w") as json_file:
json.dump(dict(self.h1_word_frequency), json_file, indent=4)
print(
f"Number of pages crawled vs left to be crawled -> {self.counter.value}/{len(self.url_queue)} = {self.counter.value / len(self.url_queue)} "
)
plt.plot(range(1, total_pages_processed + 1), pages_per_second_list)
plt.xlabel("Pages")
plt.ylabel("Pages per Second")
plt.title("Pages per Second for each page")
plt.savefig("pages_per_second.png")
speed_table = pd.DataFrame(
{
"Total Pages Crawled": [total_pages_crawled],
"Total Crawl Time (seconds)": [total_crawl_time],
"Pages per Minute": [pages_per_minute],
}
)
print("\nCrawl Speed in terms of Pages per Minute:")
print(speed_table)
fig, ax = plt.subplots(figsize=(8, 4))
ax.axis("tight")
ax.axis("off")
ax.table(
cellText=speed_table.values,
colLabels=speed_table.columns,
cellLoc="center",
loc="center",
)
plt.savefig("crawlspeed.png")
crawl_ratio_table = pd.DataFrame(
{
"Total Pages Crawled": [total_pages_crawled],
"Pages Left to Crawl": [len(self.url_queue)],
"Crawl Ratio": [{self.counter.value / len(self.url_queue)}],
}
)
fig, ax = plt.subplots(figsize=(8, 4))
ax.axis("tight")
ax.axis("off")
ax.table(
cellText=crawl_ratio_table.values,
colLabels=crawl_ratio_table.columns,
cellLoc="center",
loc="center",
)
plt.savefig("crawl_ratio_table.png")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Crawler to crawl forward from a seed URL")
parser = argparse.ArgumentParser(
description="Crawler to crawl forward from a seed URL"
)
parser.add_argument("seed_url", help="URL to start the crawling from")
parser.add_argument(
"--pages_to_parse",

35
README.md
View File

@@ -1 +1,36 @@
# webcrawler
## Instructions to run
1. Clone the repository
```
git clone https://github.com/AmanTahiliani/webcrawler.git
```
2. cd into the directory
```
cd webcrawler
```
3. Install the requirements
```
pip install -r requirements.txt
```
4. Run the shell script
```
./amantahiliani.sh
```
Note: If you get a permission denied error, run the following command
```
chmod +x amantahiliani.sh
```
If it still doesn't work, run the following command
```
python3 MultiThreadedCrawler.py {seed_url} --pages_to_parse=1000
```
## Output
The output is generated in the form of the following files:
1. pages_per_second.png - A graph showing the number of pages crawled per second
2. craw_ratio_table.png- A table showing the ratio of pages crawled to pages discovered
3. crawlspeed.png- A graph showing the crawl speed in pages per minute
4. Keywords_Output.json- A json file containing the keywords and the urls they were found in along with the frequency of the keyword in the url

12
SingleThreaded.py
View File

@@ -34,15 +34,19 @@ class WebCrawler:
for page_link in all_page_links:
self.url_queue.append(urljoin(url, page_link["href"]))
filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer","nav"]))
filtered_elements = soup.find_all(
lambda tag: tag.name in ["h1", "h2", "h3"]
and not tag.find_parents(["header", "footer", "nav"])
)
for tags in filtered_elements:
current_sentence = tags.text
for current_word in current_sentence.split():
current_word = current_word.lower()
current_word_dict = self.h1_word_frequency.get(current_word, {url:0})
current_word_dict = self.h1_word_frequency.get(
current_word, {url: 0}
)
current_word_dict[url] = current_word_dict.get(url, 0) + 1
self.h1_word_frequency[current_word] = current_word_dict
@@ -83,7 +87,7 @@ class WebCrawler:
print("Done Parsing Pages")
print(f"Pages Parsed {len(self.visited_urls)}/{pages_to_parse}")
print(f'Words found in h1 {len(self.h1_word_frequency)}')
print(f"Words found in h1 {len(self.h1_word_frequency)}")
print(self.h1_word_frequency)
# print(f'Words found in h2 {len(self.h2_word_frequency)}')
# print(self.h2_word_frequency)

6
amantahiliani.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/bash
url="https://www.cc.gatech.edu"
pages_to_parse=1000
python3 MultiThreaded.py "$url" --pages_to_parse="$pages_to_parse"