No crawler, no sitemap

2025-11-30 09:03:15 +00:00 · 2023-10-12 01:03:40 +04:00
parent 8f2ad38503
commit 658867cb46
7 changed files with 76 additions and 1 deletions
--- a/application/parser/remote/web_loader.py
+++ b/application/parser/remote/web_loader.py
@@ -0,0 +1,10 @@
+from application.parser.remote.base import BaseRemote
+
+class WebLoader(BaseRemote):
+    def __init__(self):
+        from langchain.document_loaders import WebBaseLoader
+        self.loader = WebBaseLoader
+
+    def load_data(self, urls):
+        loader = self.loader(urls)
+        return loader.load()