(refactor:upload) simplify call to /api/remote

application/api/user/routes.py

@@ -3,6 +3,7 @@ import math
 import os
 import shutil
 import uuid
+import json
 
 from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
@@ -420,18 +421,22 @@ class UploadRemote(Resource):
             return missing_fields
 
         try:
-            if "repo_url" in data:
-                source_data = data["repo_url"]
-                loader = "github"
-            else:
-                source_data = data["data"]
-                loader = data["source"]
+           config = json.loads(data["data"])
+           source_data = None
+
+           if data["source"] == "github":
+                source_data = config.get("repo_url")
+           elif data["source"] in ["crawler", "url"]:
+                source_data = config.get("url")
+           elif data["source"] == "reddit":
+                source_data = config 
+
 
            task = ingest_remote.delay(
                 source_data=source_data,
                 job_name=data["name"],
                 user=data["user"],
-                loader=loader,
+                loader=data["source"]
             )
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)

frontend/src/upload/Upload.tsx

@@ -20,10 +20,6 @@ import WrapperModal from '../modals/WrapperModal';
 import {
   IngestorType,
   IngestorConfig,
-  RedditIngestorConfig,
-  GithubIngestorConfig,
-  CrawlerIngestorConfig,
-  UrlIngestorConfig,
   IngestorFormSchemas,
   FormField,
 } from './types/ingestor';
@@ -397,20 +393,7 @@ function Upload({
     formData.append('user', 'local');
     formData.append('source', ingestor.type);
 
-    if (ingestor.type === 'reddit') {
-      const redditConfig = ingestor.config as RedditIngestorConfig;
-
-      formData.set('data', JSON.stringify(redditConfig));
-    } else if (ingestor.type === 'github') {
-      const githubConfig = ingestor.config as GithubIngestorConfig;
-      formData.append('repo_url', githubConfig.repo_url);
-      formData.append('data', githubConfig.repo_url);
-    } else {
-      const urlBasedConfig = ingestor.config as
-        | CrawlerIngestorConfig
-        | UrlIngestorConfig;
-      formData.append('data', urlBasedConfig.url);
-    }
+    formData.append('data', JSON.stringify(ingestor.config));
 
     const apiHost: string = import.meta.env.VITE_API_HOST;
     const xhr = new XMLHttpRequest();