LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 08:33:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

(fix:driveLoader) folder ingesting

Browse Source
This commit is contained in:
ManishMadan2882
2025-08-22 19:07:52 +05:30
parent b1210c4902
commit 2410bd8654
4 changed files with 119 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
application/worker.py
View File

@@ -854,11 +854,11 @@ def ingest_connector(
"""
logging.info(f"Starting remote ingestion from {source_type} for user: {user}, job: {job_name}")
self.update_state(state="PROGRESS", meta={"current": 1})
with tempfile.TemporaryDirectory() as temp_dir:
try:
# Step 1: Get the appropriate remote loader
logging.info(f"source_config {source_config}")
# Step 1: Initialize the appropriate loader
self.update_state(state="PROGRESS", meta={"current": 10, "status": "Initializing connector"})
if source_type == "google_drive":
session_token = source_config.get("session_token")
@@ -871,11 +871,10 @@ def ingest_connector(
# Create a clean config for storage that excludes the session token
api_source_config = {
"file_ids": source_config.get("file_ids", []),
"folder_id": source_config.get("folder_id", ""),
"folder_ids": source_config.get("folder_ids", []),
"recursive": source_config.get("recursive", True)
}
if source_config.get("recursive") is not None:
api_source_config["recursive"] = source_config.get("recursive")
else:
remote_loader = RemoteCreator.create_loader(source_type, source_config)
api_source_config = source_config