diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 10b141c0..6c18a680 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -3,6 +3,7 @@ import math import os import shutil import uuid +import json from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef @@ -420,18 +421,22 @@ class UploadRemote(Resource): return missing_fields try: - if "repo_url" in data: - source_data = data["repo_url"] - loader = "github" - else: - source_data = data["data"] - loader = data["source"] + config = json.loads(data["data"]) + source_data = None - task = ingest_remote.delay( + if data["source"] == "github": + source_data = config.get("repo_url") + elif data["source"] in ["crawler", "url"]: + source_data = config.get("url") + elif data["source"] == "reddit": + source_data = config + + + task = ingest_remote.delay( source_data=source_data, job_name=data["name"], user=data["user"], - loader=loader, + loader=data["source"] ) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 78e9a321..7eb569ef 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -20,10 +20,6 @@ import WrapperModal from '../modals/WrapperModal'; import { IngestorType, IngestorConfig, - RedditIngestorConfig, - GithubIngestorConfig, - CrawlerIngestorConfig, - UrlIngestorConfig, IngestorFormSchemas, FormField, } from './types/ingestor'; @@ -397,20 +393,7 @@ function Upload({ formData.append('user', 'local'); formData.append('source', ingestor.type); - if (ingestor.type === 'reddit') { - const redditConfig = ingestor.config as RedditIngestorConfig; - - formData.set('data', JSON.stringify(redditConfig)); - } else if (ingestor.type === 'github') { - const githubConfig = ingestor.config as GithubIngestorConfig; - formData.append('repo_url', githubConfig.repo_url); - formData.append('data', githubConfig.repo_url); - } else { - const urlBasedConfig = ingestor.config as - | CrawlerIngestorConfig - | UrlIngestorConfig; - formData.append('data', urlBasedConfig.url); - } + formData.append('data', JSON.stringify(ingestor.config)); const apiHost: string = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest();