Desc loader

2025-11-29 08:33:20 +00:00 · 2023-10-12 13:44:32 +04:00
parent 50f07f9ef5
commit 2cfb416fd0
1 changed files with 1 additions and 0 deletions
--- a/application/parser/remote/crawler_loader.py
+++ b/application/parser/remote/crawler_loader.py
@@ -7,6 +7,7 @@ class CrawlerLoader(BaseRemote):
    def __init__(self, limit=10):
        from langchain.document_loaders import WebBaseLoader
        self.loader = WebBaseLoader
+        #No pages scraped limit, set None for no limit
        self.limit = limit

    def load_data(self, url):