fix: connection aborted in WebBaseLoader

application/parser/remote/web_loader.py

@@ -1,22 +1,23 @@
 from application.parser.remote.base import BaseRemote
+from langchain_community.document_loaders import WebBaseLoader
+
 
 class WebLoader(BaseRemote):
     def __init__(self):
-        from langchain.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader
 
     def load_data(self, inputs):
         urls = inputs
-
         if isinstance(urls, str):
-            urls = [urls] # Convert string to list if a single URL is passed
-
+            urls = [urls]
         documents = []
         for url in urls:
             try:
-                loader = self.loader([url])  # Process URLs one by one
+                loader = self.loader(
+                    [url], header_template={"User-Agent": "Mozilla/5.0"}
+                )
                 documents.extend(loader.load())
             except Exception as e:
                 print(f"Error processing URL {url}: {e}")
-                continue  # Continue with the next URL if an error occurs
-        return documents
+                continue
+        return documents