amantahiliani/webcrawler
1
0
Fork 0
mirror of https://github.com/AmanTahiliani/webcrawler.git synced 2026-08-07 11:55:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

Updated Output code and Readme

Browse Source
This commit is contained in:
Aman Tahiliani
2024-01-26 19:31:47 -05:00
parent 129f0f7542
commit f510e000d7
4 changed files with 144 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
SingleThreaded.py
View File

@@ -34,15 +34,19 @@ class WebCrawler:
for page_link in all_page_links:
self.url_queue.append(urljoin(url, page_link["href"]))
filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer","nav"]))
filtered_elements = soup.find_all(
lambda tag: tag.name in ["h1", "h2", "h3"]
and not tag.find_parents(["header", "footer", "nav"])
)
for tags in filtered_elements:
current_sentence = tags.text
for current_word in current_sentence.split():
current_word = current_word.lower()
current_word_dict = self.h1_word_frequency.get(current_word, {url:0})
current_word_dict = self.h1_word_frequency.get(
current_word, {url: 0}
)
current_word_dict[url] = current_word_dict.get(url, 0) + 1
self.h1_word_frequency[current_word] = current_word_dict
@@ -83,7 +87,7 @@ class WebCrawler:
print("Done Parsing Pages")
print(f"Pages Parsed {len(self.visited_urls)}/{pages_to_parse}")
print(f'Words found in h1 {len(self.h1_word_frequency)}')
print(f"Words found in h1 {len(self.h1_word_frequency)}")
print(self.h1_word_frequency)
# print(f'Words found in h2 {len(self.h2_word_frequency)}')
# print(self.h2_word_frequency)