diff --git a/.gitignore b/.gitignore index 68bc17f..f7275bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,160 +1 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ diff --git a/MultiThreaded.py b/MultiThreaded.py new file mode 100644 index 0000000..b66f562 --- /dev/null +++ b/MultiThreaded.py @@ -0,0 +1,111 @@ +__author__= "Aman Tahiliani" + +import requests +from bs4 import BeautifulSoup as bs +from urllib.parse import urljoin +import argparse +import multiprocessing + +class WebCrawler: + def __init__(self, seed_url): + manager = multiprocessing.Manager() + + self.url_queue = manager.list() + self.url_queue.append(seed_url) + self.visited_urls = manager.list() + self.counter = manager.Value('i', 0) + self.h1_word_frequency = manager.dict() + + # Using manager to create locks + self.url_queue_lock = manager.Lock() + self.visited_lock = manager.Lock() + self.freq_lock = manager.Lock() + + def extract_page_info(self, url): + # print(f"Extracting information from Url {url}") + try: + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE" + } + try: + response = requests.get(url, headers=headers) + except: + return + + if response.status_code != 200: + raise Exception(response.text) + + soup = bs(response.content, "html.parser") + all_page_links = soup.find_all("a", href=True) + # print(f"Found {len(set(all_page_links))} links in the current page") + for page_link in all_page_links: + with self.url_queue_lock: + self.url_queue.append(urljoin(url, page_link["href"])) + + filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer", "nav"])) + + for tags in filtered_elements: + current_sentence = tags.text + for current_word in current_sentence.split(): + current_word = current_word.lower() + + with self.freq_lock: + current_word_dict = self.h1_word_frequency.get(current_word, {url: 0}) + current_word_dict[url] = current_word_dict.get(url, 0) + 1 + self.h1_word_frequency[current_word] = current_word_dict + + return True + + except Exception as e: + print( + f"Exception occurred while crawling page {url}. Exception -> {e}" + ) + return False + + def process_url(self, new_url): + shouldVisit = False + with self.visited_lock: + if new_url not in self.visited_urls: + shouldVisit = True + self.visited_urls.append(new_url) + self.counter.value += 1 + print(f"Page Number {self.counter.value}") + if shouldVisit: + visited = self.extract_page_info(new_url) + + def crawler(self, pages_to_parse): + with multiprocessing.Pool() as pool: + while True: + with self.url_queue_lock: + if not self.url_queue: + break + chunk_size = min(len(self.url_queue), multiprocessing.cpu_count()) + chunks = [self.url_queue.pop(0) for chunk in range(chunk_size)] + pool.starmap(self.process_url, [(url,) for url in chunks]) + + with self.visited_lock: + counter_val = self.counter.value + if counter_val >= pages_to_parse: + break + + print("Done Parsing Pages") + print(f"Pages Parsed {self.counter.value}/{pages_to_parse}") + print(f'Links left to parse {len(self.url_queue)}') + print(f'Words found in h1 {len(self.h1_word_frequency)}') + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Crawler to crawl forward from a seed URL") + parser.add_argument("seed_url", help="URL to start the crawling from") + parser.add_argument( + "--pages_to_parse", + help="Upper limit of how many pages that need to be parsed", + default=1000, + ) + args = parser.parse_args() + seed_url = args.seed_url + pages_to_parse = int(args.pages_to_parse) + + web_crawler = WebCrawler(seed_url) + print(f"Seed Url: {seed_url} \nNumber of Pages to Parse {pages_to_parse}") + print("Initiating Crawler....") + web_crawler.crawler(pages_to_parse) \ No newline at end of file diff --git a/SingleThreaded.py b/SingleThreaded.py new file mode 100644 index 0000000..02b6e71 --- /dev/null +++ b/SingleThreaded.py @@ -0,0 +1,110 @@ +__author__ = "Aman Tahiliani" + +import requests +import lxml +from bs4 import BeautifulSoup as bs +from urllib.parse import urljoin +import argparse + + +class WebCrawler: + def __init__(self): + self.title_keyword_frequency = {} + self.word_frequency = {} + self.url_queue = [seed_url] + self.visited_urls = set() + self.counter = 0 + + self.h1_word_frequency = {} + + def extract_page_info(self, url): + print(f"Extracting information from Url {url}") + try: + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE" + } + response = requests.get(url, headers=headers) + + if response.status_code != 200: + raise Exception(response.text) + + soup = bs(response.content, "html.parser") + all_page_links = soup.find_all("a", href=True) + print(f"Found {len(all_page_links)} links in the current page") + for page_link in all_page_links: + self.url_queue.append(urljoin(url, page_link["href"])) + + filtered_elements = soup.find_all(lambda tag: tag.name in ["h1", "h2", "h3"] and not tag.find_parents(["header", "footer","nav"])) + + + for tags in filtered_elements: + current_sentence = tags.text + for current_word in current_sentence.split(): + current_word = current_word.lower() + + current_word_dict = self.h1_word_frequency.get(current_word, {url:0}) + current_word_dict[url] = current_word_dict.get(url, 0) + 1 + self.h1_word_frequency[current_word] = current_word_dict + + # for tags in h2_tags: + # current_sentence = tags.text + # for current_word in current_sentence.split(): + # self.h2_word_frequency[current_word] = ( + # self.h2_word_frequency.get(current_word, 0) + 1 + # ) + return True + + except Exception as e: + print( + "Exception occured while crawling page {url}. Exception -> {e}".format( + url=url, e=e + ) + ) + return False + + def crawler(self, seed_url, pages_to_parse): + # Clearning the frequency Dictionaries + self.title_keyword_frequency.clear() + self.word_frequency.clear() + + self.url_queue = [seed_url] + self.visited_urls = set() + self.counter = 0 + + while len(self.url_queue) and self.counter < pages_to_parse: + new_url = self.url_queue.pop(0) + + if new_url not in self.visited_urls: + self.counter += 1 + print(f"Page Numeber {self.counter}") + visited = self.extract_page_info(new_url) + if visited: + self.visited_urls.add(new_url) + + print("Done Parsing Pages") + print(f"Pages Parsed {len(self.visited_urls)}/{pages_to_parse}") + print(f'Words found in h1 {len(self.h1_word_frequency)}') + print(self.h1_word_frequency) + # print(f'Words found in h2 {len(self.h2_word_frequency)}') + # print(self.h2_word_frequency) + + +parser = argparse.ArgumentParser(description="Crawler to crawl forward from a seed URL") +parser.add_argument("seed_url", help="URL to start the crawling from") +parser.add_argument( + "--pages_to_parse", + help="Upper limit of how many pages that need to be parsed", + default=1000, +) +args = parser.parse_args() +seed_url = args.seed_url +pages_to_parse = args.pages_to_parse + + +if type(pages_to_parse == str): + pages_to_parse = int(pages_to_parse) + +web_crawler = WebCrawler() +print(f"Seed Url: {seed_url} \n Number of Pages to Parse {pages_to_parse}") +print("Initiating Crawler....") +web_crawler.crawler(seed_url, pages_to_parse)