This commit is contained in:
Pavel
2023-10-12 19:40:23 +04:00
parent 2cfb416fd0
commit 719ca63ec1
4 changed files with 109 additions and 46 deletions

View File

@@ -6,5 +6,15 @@ class WebLoader(BaseRemote):
self.loader = WebBaseLoader
def load_data(self, urls):
loader = self.loader(urls)
return loader.load()
if isinstance(urls, str):
urls = [urls] # Convert string to list if a single URL is passed
documents = []
for url in urls:
try:
loader = self.loader([url]) # Process URLs one by one
documents.extend(loader.load())
except Exception as e:
print(f"Error processing URL {url}: {e}")
continue # Continue with the next URL if an error occurs
return documents