LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 08:33:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix wrong link

Browse Source
This commit is contained in:
Pavel
2023-10-12 19:45:36 +04:00
parent 719ca63ec1
commit b7d88b4c0f
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
application/parser/remote/sitemap_loader.py
View File

@@ -33,9 +33,11 @@ class SitemapLoader(BaseRemote):
return documents
def _extract_urls(self, sitemap_url):
response = requests.get(sitemap_url)
if response.status_code != 200:
print(f"Failed to fetch sitemap: {sitemap_url}")
try:
response = requests.get(sitemap_url)
response.raise_for_status() # Raise an exception for HTTP errors
except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
print(f"Failed to fetch sitemap: {sitemap_url}. Error: {e}")
return []
# Determine if this is a sitemap or a URL