amantahiliani/webcrawler
1
0
Fork 0
mirror of https://github.com/AmanTahiliani/webcrawler.git synced 2026-08-07 11:55:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

Updated Output code and Readme

Browse Source
This commit is contained in:
Aman Tahiliani
2024-01-26 19:31:47 -05:00
parent 129f0f7542
commit f510e000d7
4 changed files with 144 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
MultiThreaded.py
View File

@@ -5,6 +5,11 @@ from bs4 import BeautifulSoup as bs
from urllib.parse import urljoin
import argparse
import multiprocessing
import time
import matplotlib.pyplot as plt
import pandas as pd
import json
class WebCrawler:
def __init__(self, seed_url):
@@ -13,10 +18,9 @@ class WebCrawler:
self.url_queue = manager.list()
self.url_queue.append(seed_url)
self.visited_urls = manager.list()
self.counter = manager.Value('i', 0)
self.counter = manager.Value("i", 0)
self.h1_word_frequency = manager.dict()
# Using manager to create locks
self.url_queue_lock = manager.Lock()
self.visited_lock = manager.Lock()
self.freq_lock = manager.Lock()
@@ -42,7 +46,10 @@ class WebCrawler:
with self.url_queue_lock:
self.url_queue.append(urljoin(url, page_link["href"]))
filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer", "nav"]))
filtered_elements = soup.find_all(
lambda tag: tag.name in ["h1", "h2", "h3"]
and not tag.find_parents(["header", "footer", "nav"])
)
for tags in filtered_elements:
current_sentence = tags.text
@@ -50,16 +57,16 @@ class WebCrawler:
current_word = current_word.lower()
with self.freq_lock:
current_word_dict = self.h1_word_frequency.get(current_word, {url: 0})
current_word_dict = self.h1_word_frequency.get(
current_word, {url: 0}
)
current_word_dict[url] = current_word_dict.get(url, 0) + 1
self.h1_word_frequency[current_word] = current_word_dict
return True
except Exception as e:
print(
f"Exception occurred while crawling page {url}. Exception -> {e}"
)
print(f"Exception occurred while crawling page {url}. Exception -> {e}")
return False
def process_url(self, new_url):
@@ -74,6 +81,9 @@ class WebCrawler:
visited = self.extract_page_info(new_url)
def crawler(self, pages_to_parse):
crawl_start_time = time.time()
pages_per_second_list = []
total_pages_processed = 0
with multiprocessing.Pool() as pool:
while True:
with self.url_queue_lock:
@@ -81,20 +91,91 @@ class WebCrawler:
break
chunk_size = min(len(self.url_queue), multiprocessing.cpu_count())
chunks = [self.url_queue.pop(0) for chunk in range(chunk_size)]
batch_start_time = time.time()
pool.starmap(self.process_url, [(url,) for url in chunks])
batch_end_time = time.time()
pages_processed = len(chunks)
total_pages_processed += pages_processed
time_taken = batch_end_time - batch_start_time
pages_per_second = pages_processed / time_taken
for _ in range(pages_processed):
pages_per_second_list.append(pages_per_second)
with self.visited_lock:
counter_val = self.counter.value
if counter_val >= pages_to_parse:
break
crawl_end_time = time.time()
total_crawl_time = crawl_end_time - crawl_start_time
total_pages_crawled = self.counter.value
pages_per_minute = total_pages_crawled / (total_crawl_time / 60)
print("Done Parsing Pages")
print(f"Pages Parsed {self.counter.value}/{pages_to_parse}")
print(f'Links left to parse {len(self.url_queue)}')
print(f'Words found in h1 {len(self.h1_word_frequency)}')
print(f"Links left to parse {len(self.url_queue)}")
print(f"Words found in h1 {len(self.h1_word_frequency)}")
with open("Keywords_Output.json", "w") as json_file:
json.dump(dict(self.h1_word_frequency), json_file, indent=4)
print(
f"Number of pages crawled vs left to be crawled -> {self.counter.value}/{len(self.url_queue)} = {self.counter.value / len(self.url_queue)} "
)
plt.plot(range(1, total_pages_processed + 1), pages_per_second_list)
plt.xlabel("Pages")
plt.ylabel("Pages per Second")
plt.title("Pages per Second for each page")
plt.savefig("pages_per_second.png")
speed_table = pd.DataFrame(
{
"Total Pages Crawled": [total_pages_crawled],
"Total Crawl Time (seconds)": [total_crawl_time],
"Pages per Minute": [pages_per_minute],
}
)
print("\nCrawl Speed in terms of Pages per Minute:")
print(speed_table)
fig, ax = plt.subplots(figsize=(8, 4))
ax.axis("tight")
ax.axis("off")
ax.table(
cellText=speed_table.values,
colLabels=speed_table.columns,
cellLoc="center",
loc="center",
)
plt.savefig("crawlspeed.png")
crawl_ratio_table = pd.DataFrame(
{
"Total Pages Crawled": [total_pages_crawled],
"Pages Left to Crawl": [len(self.url_queue)],
"Crawl Ratio": [{self.counter.value / len(self.url_queue)}],
}
)
fig, ax = plt.subplots(figsize=(8, 4))
ax.axis("tight")
ax.axis("off")
ax.table(
cellText=crawl_ratio_table.values,
colLabels=crawl_ratio_table.columns,
cellLoc="center",
loc="center",
)
plt.savefig("crawl_ratio_table.png")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Crawler to crawl forward from a seed URL")
parser = argparse.ArgumentParser(
description="Crawler to crawl forward from a seed URL"
)
parser.add_argument("seed_url", help="URL to start the crawling from")
parser.add_argument(
"--pages_to_parse",

35
README.md
View File

@@ -1 +1,36 @@
# webcrawler
## Instructions to run
1. Clone the repository
```
git clone https://github.com/AmanTahiliani/webcrawler.git
```
2. cd into the directory
```
cd webcrawler
```
3. Install the requirements
```
pip install -r requirements.txt
```
4. Run the shell script
```
./amantahiliani.sh
```
Note: If you get a permission denied error, run the following command
```
chmod +x amantahiliani.sh
```
If it still doesn't work, run the following command
```
python3 MultiThreadedCrawler.py {seed_url} --pages_to_parse=1000
```
## Output
The output is generated in the form of the following files:
1. pages_per_second.png - A graph showing the number of pages crawled per second
2. craw_ratio_table.png- A table showing the ratio of pages crawled to pages discovered
3. crawlspeed.png- A graph showing the crawl speed in pages per minute
4. Keywords_Output.json- A json file containing the keywords and the urls they were found in along with the frequency of the keyword in the url

12
SingleThreaded.py
View File

@@ -34,15 +34,19 @@ class WebCrawler:
for page_link in all_page_links:
self.url_queue.append(urljoin(url, page_link["href"]))
filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer","nav"]))
filtered_elements = soup.find_all(
lambda tag: tag.name in ["h1", "h2", "h3"]
and not tag.find_parents(["header", "footer", "nav"])
)
for tags in filtered_elements:
current_sentence = tags.text
for current_word in current_sentence.split():
current_word = current_word.lower()
current_word_dict = self.h1_word_frequency.get(current_word, {url:0})
current_word_dict = self.h1_word_frequency.get(
current_word, {url: 0}
)
current_word_dict[url] = current_word_dict.get(url, 0) + 1
self.h1_word_frequency[current_word] = current_word_dict
@@ -83,7 +87,7 @@ class WebCrawler:
print("Done Parsing Pages")
print(f"Pages Parsed {len(self.visited_urls)}/{pages_to_parse}")
print(f'Words found in h1 {len(self.h1_word_frequency)}')
print(f"Words found in h1 {len(self.h1_word_frequency)}")
print(self.h1_word_frequency)
# print(f'Words found in h2 {len(self.h2_word_frequency)}')
# print(self.h2_word_frequency)

6
amantahiliani.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/bash
url="https://www.cc.gatech.edu"
pages_to_parse=1000
python3 MultiThreaded.py "$url" --pages_to_parse="$pages_to_parse"