mirror of
https://github.com/arc53/DocsGPT.git
synced 2025-11-29 16:43:16 +00:00
web loader fix
Changes web loader to the correct output.
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
from application.parser.remote.base import BaseRemote
|
||||
from application.parser.schema.base import Document
|
||||
from langchain_community.document_loaders import WebBaseLoader
|
||||
from urllib.parse import urlparse
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
@@ -23,10 +25,20 @@ class WebLoader(BaseRemote):
|
||||
urls = [urls]
|
||||
documents = []
|
||||
for url in urls:
|
||||
# Check if the URL scheme is provided, if not, assume http
|
||||
if not urlparse(url).scheme:
|
||||
url = "http://" + url
|
||||
try:
|
||||
loader = self.loader([url], header_template=headers)
|
||||
documents.extend(loader.load())
|
||||
loaded_docs = loader.load()
|
||||
for doc in loaded_docs:
|
||||
documents.append(
|
||||
Document(
|
||||
doc.page_content,
|
||||
extra_info=doc.metadata,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error processing URL {url}: {e}")
|
||||
continue
|
||||
return documents
|
||||
return documents
|
||||
Reference in New Issue
Block a user