From b7d88b4c0f56f1a0197930be0da0a3080b089e4e Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 12 Oct 2023 19:45:36 +0400 Subject: [PATCH] fix wrong link --- application/parser/remote/sitemap_loader.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/application/parser/remote/sitemap_loader.py b/application/parser/remote/sitemap_loader.py index 0a3f4d4c..a8700555 100644 --- a/application/parser/remote/sitemap_loader.py +++ b/application/parser/remote/sitemap_loader.py @@ -33,9 +33,11 @@ class SitemapLoader(BaseRemote): return documents def _extract_urls(self, sitemap_url): - response = requests.get(sitemap_url) - if response.status_code != 200: - print(f"Failed to fetch sitemap: {sitemap_url}") + try: + response = requests.get(sitemap_url) + response.raise_for_status() # Raise an exception for HTTP errors + except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e: + print(f"Failed to fetch sitemap: {sitemap_url}. Error: {e}") return [] # Determine if this is a sitemap or a URL