diff --git a/application/parser/remote/crawler_loader.py b/application/parser/remote/crawler_loader.py index e69de29b..208195cb 100644 --- a/application/parser/remote/crawler_loader.py +++ b/application/parser/remote/crawler_loader.py @@ -0,0 +1,36 @@ +import requests +from urllib.parse import urlparse, urljoin +from bs4 import BeautifulSoup +from application.parser.remote.base import BaseRemote + +class CrawlerLoader(BaseRemote): + def __init__(self): + from langchain.document_loaders import WebBaseLoader + self.loader = WebBaseLoader + + def load_data(self, url): + # Fetch the content of the initial URL + response = requests.get(url) + if response.status_code != 200: + print(f"Failed to fetch initial URL: {url}") + return None + + # Parse the HTML content + soup = BeautifulSoup(response.text, 'html.parser') + + # Extract the base URL to ensure we only fetch URLs from the same domain + base_url = urlparse(url).scheme + "://" + urlparse(url).hostname + + # Extract all links from the HTML content + all_links = [a['href'] for a in soup.find_all('a', href=True)] + + # Filter out the links that lead to a different domain + same_domain_links = [urljoin(base_url, link) for link in all_links if base_url in urljoin(base_url, link)] + + # Remove duplicates + same_domain_links = list(set(same_domain_links)) + + #TODO: Optimize this section to parse pages as they are being crawled + loaded_content = self.loader(same_domain_links).load() + + return loaded_content diff --git a/application/parser/remote/sitemap_loader.py b/application/parser/remote/sitemap_loader.py index e69de29b..366d81ed 100644 --- a/application/parser/remote/sitemap_loader.py +++ b/application/parser/remote/sitemap_loader.py @@ -0,0 +1,27 @@ +import requests +import xml.etree.ElementTree as ET +from application.parser.remote.base import BaseRemote + +class SitemapLoader(BaseRemote): + def __init__(self): + from langchain.document_loaders import WebBaseLoader + self.loader = WebBaseLoader + + def load_data(self, sitemap_url): + # Fetch the sitemap content + response = requests.get(sitemap_url) + if response.status_code != 200: + print(f"Failed to fetch sitemap: {sitemap_url}") + return None + + # Parse the sitemap XML + root = ET.fromstring(response.content) + + # Extract URLs from the sitemap + # The namespace with "loc" tag might be needed to extract URLs + ns = {'s': 'http://www.sitemaps.org/schemas/sitemap/0.9'} + urls = [loc.text for loc in root.findall('s:url/s:loc', ns)] + + # Use your existing loader to load content of extracted URLs + loader = self.loader(urls) + return loader.load() \ No newline at end of file