(refactor:remote uploads) dynamic ingestor types

frontend/src/components/ToggleSwitch.tsx

@@ -0,0 +1,58 @@
+import React from 'react';
+
+type ToggleSwitchProps = {
+  checked: boolean;
+  onChange: (checked: boolean) => void;
+  className?: string;
+  label?: string;
+  disabled?: boolean;
+  activeColor?: string;
+  inactiveColor?: string;
+  id?: string;
+};
+
+const ToggleSwitch: React.FC<ToggleSwitchProps> = ({
+  checked,
+  onChange,
+  className = '',
+  label,
+  disabled = false,
+  activeColor = 'bg-purple-30',
+  inactiveColor = 'bg-transparent',
+  id,
+}) => {
+  return (
+    <label
+      className={`cursor-pointer select-none items-center ${disabled ? 'opacity-50 cursor-not-allowed' : ''} ${className}`}
+      htmlFor={id}
+    >
+      <div className="relative">
+        <input
+          type="checkbox"
+          checked={checked}
+          onChange={(e) => onChange(e.target.checked)}
+          className="sr-only"
+          disabled={disabled}
+          id={id}
+        />
+        <div
+          className={`box block h-8 w-14 rounded-full border border-purple-30 ${
+            checked
+              ? `${activeColor} dark:${activeColor}`
+              : `${inactiveColor} dark:${inactiveColor}`
+          }`}
+        ></div>
+        <div
+          className={`absolute left-1 top-1 flex h-6 w-6 items-center justify-center rounded-full transition ${
+            checked ? 'translate-x-full bg-silver' : 'bg-purple-30'
+          }`}
+        ></div>
+      </div>
+      {label && (
+        <span className="ml-2 text-eerie-black dark:text-white">{label}</span>
+      )}
+    </label>
+  );
+};
+
+export default ToggleSwitch;

frontend/src/upload/Upload.tsx

@@ -16,6 +16,23 @@ import {
   selectSourceDocs,
 } from '../preferences/preferenceSlice';
 import WrapperModal from '../modals/WrapperModal';
+import {
+  IngestorType,
+  IngestorConfig,
+  RedditIngestorConfig,
+  GithubIngestorConfig,
+  CrawlerIngestorConfig,
+  UrlIngestorConfig,
+} from './types/ingestor';
+
+type IngestorState = {
+  type: IngestorType;
+  config:
+    | RedditIngestorConfig
+    | GithubIngestorConfig
+    | CrawlerIngestorConfig
+    | UrlIngestorConfig;
+};
 
 function Upload({
   receivedFile = [],
@@ -33,18 +50,19 @@ function Upload({
   onSuccessfulUpload?: () => void;
 }) {
   const [docName, setDocName] = useState(receivedFile[0]?.name);
-  const [urlName, setUrlName] = useState('');
-  const [url, setUrl] = useState('');
-  const [repoUrl, setRepoUrl] = useState(''); // P3f93
-  const [redditData, setRedditData] = useState({
-    client_id: '',
-    client_secret: '',
-    user_agent: '',
-    search_queries: [''],
-    number_posts: 10,
-  });
-  const [activeTab, setActiveTab] = useState<string | null>(renderTab);
   const [files, setfiles] = useState<File[]>(receivedFile);
+  const [activeTab, setActiveTab] = useState<string | null>(renderTab);
+
+  // New unified ingestor state
+  const [ingestor, setIngestor] = useState<IngestorConfig>({
+    type: 'crawler',
+    name: '',
+    config: {
+      name: '',
+      url: '',
+    } as CrawlerIngestorConfig,
+  });
+
   const [progress, setProgress] = useState<{
     type: 'UPLOAD' | 'TRAINING';
     percentage: number;
@@ -55,12 +73,11 @@ function Upload({
   const { t } = useTranslation();
   const setTimeoutRef = useRef<number | null>();
 
-  const urlOptions: { label: string; value: string }[] = [
+  const urlOptions: { label: string; value: IngestorType }[] = [
-    { label: `Crawler`, value: 'crawler' },
+    { label: 'Crawler', value: 'crawler' },
-    // { label: t('modals.uploadDoc.sitemap'), value: 'sitemap' },
+    { label: 'Link', value: 'url' },
-    { label: `Link`, value: 'url' },
+    { label: 'GitHub', value: 'github' },
-    { label: `GitHub`, value: 'github' },
+    { label: 'Reddit', value: 'reddit' },
-    { label: `Reddit`, value: 'reddit' },
   ];
 
   const [urlType, setUrlType] = useState<{ label: string; value: string }>({
@@ -284,22 +301,23 @@ function Upload({
 
   const uploadRemote = () => {
     const formData = new FormData();
-    formData.append('name', urlName);
+    formData.append('name', ingestor.name);
     formData.append('user', 'local');
-    if (urlType !== null) {
+    formData.append('source', ingestor.type);
-      formData.append('source', urlType?.value);
+
-    }
+    if (ingestor.type === 'reddit') {
-    formData.append('data', url);
+      formData.set('data', JSON.stringify(ingestor.config));
-    if (
+    } else if (ingestor.type === 'github') {
-      redditData.client_id.length > 0 &&
+      const githubConfig = ingestor.config as GithubIngestorConfig;
-      redditData.client_secret.length > 0
+      formData.append('repo_url', githubConfig.repo_url);
-    ) {
+      formData.append('data', githubConfig.repo_url);
-      formData.set('name', 'other');
+    } else {
-      formData.set('data', JSON.stringify(redditData));
+      const urlBasedConfig = ingestor.config as
-    }
+        | CrawlerIngestorConfig
-    if (urlType.value === 'github') {
+        | UrlIngestorConfig;
-      formData.append('repo_url', repoUrl); // Pdeac
+      formData.append('data', urlBasedConfig.url);
     }
+
     const apiHost = import.meta.env.VITE_API_HOST;
     const xhr = new XMLHttpRequest();
     xhr.upload.addEventListener('progress', (event) => {
@@ -346,20 +364,158 @@ function Upload({
     },
   });
 
+  const isUploadDisabled = () => {
+    if (activeTab !== 'remote') return false;
+
+    switch (ingestor.type) {
+      case 'reddit': {
+        const redditConfig = ingestor.config as RedditIngestorConfig;
+        return (
+          !redditConfig.client_id ||
+          !redditConfig.client_secret ||
+          !redditConfig.user_agent ||
+          !redditConfig.search_queries.length ||
+          !redditConfig.number_posts
+        );
+      }
+      case 'github':
+        return !(ingestor.config as GithubIngestorConfig).repo_url;
+      default: {
+        const urlConfig = ingestor.config as
+          | CrawlerIngestorConfig
+          | UrlIngestorConfig;
+        return !urlConfig.url || !ingestor.name;
+      }
+    }
+  };
+
+  const handleIngestorChange = (
+    e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>,
+  ) => {
+    const { name, value } = e.target;
+
+    if (ingestor.type === 'reddit') {
+      const redditConfig = ingestor.config as RedditIngestorConfig;
+      setIngestor({
+        ...ingestor,
+        config: {
+          ...redditConfig,
+          [name]:
+            name === 'search_queries'
+              ? value.split(',').map((item) => item.trim())
+              : name === 'number_posts'
+                ? parseInt(value)
+                : value,
+        },
+      });
+    } else if (ingestor.type === 'github') {
+      const githubConfig = ingestor.config as GithubIngestorConfig;
+      setIngestor({
+        ...ingestor,
+        config: {
+          ...githubConfig,
+          [name]: value,
+        },
+      });
+    } else {
+      const urlConfig = ingestor.config as
+        | CrawlerIngestorConfig
+        | UrlIngestorConfig;
+      setIngestor({
+        ...ingestor,
+        config: {
+          ...urlConfig,
+          [name]: value,
+        },
+      });
+    }
+  };
+
   const handleChange = (
     e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>,
   ) => {
     const { name, value } = e.target;
-    if (name === 'search_queries' && value.length > 0) {
+
-      setRedditData({
+    if (ingestor.type === 'reddit') {
-        ...redditData,
+      const redditConfig = ingestor.config as RedditIngestorConfig;
-        [name]: value.split(',').map((item) => item.trim()),
+      setIngestor({
+        ...ingestor,
+        config: {
+          ...redditConfig,
+          [name]:
+            name === 'search_queries'
+              ? value.split(',').map((item) => item.trim())
+              : name === 'number_posts'
+                ? parseInt(value)
+                : value,
+        },
       });
-    } else
+    } else if (ingestor.type === 'github') {
-      setRedditData({
+      const githubConfig = ingestor.config as GithubIngestorConfig;
-        ...redditData,
+      setIngestor({
-        [name]: name === 'number_posts' ? parseInt(value) : value,
+        ...ingestor,
+        config: {
+          ...githubConfig,
+          [name]: value,
+        },
       });
+    } else {
+      const urlConfig = ingestor.config as
+        | CrawlerIngestorConfig
+        | UrlIngestorConfig;
+      setIngestor({
+        ...ingestor,
+        config: {
+          ...urlConfig,
+          [name]: value,
+        },
+      });
+    }
+  };
+
+  const handleIngestorTypeChange = (type: IngestorType) => {
+    let newConfig:
+      | RedditIngestorConfig
+      | GithubIngestorConfig
+      | CrawlerIngestorConfig
+      | UrlIngestorConfig;
+
+    switch (type) {
+      case 'reddit':
+        newConfig = {
+          name: ingestor.name,
+          client_id: '',
+          client_secret: '',
+          user_agent: '',
+          search_queries: [],
+          number_posts: 10,
+        };
+        break;
+      case 'github':
+        newConfig = {
+          name: ingestor.name,
+          repo_url: '',
+        };
+        break;
+      case 'crawler':
+      case 'url':
+        newConfig = {
+          name: ingestor.name,
+          url: '',
+        };
+        break;
+      default:
+        newConfig = {
+          name: ingestor.name,
+          url: '',
+        } as CrawlerIngestorConfig;
+    }
+
+    setIngestor({
+      type,
+      name: ingestor.name,
+      config: newConfig,
+    });
   };
 
   let view;
@@ -455,145 +611,96 @@ function Upload({
             <Dropdown
               border="border"
               options={urlOptions}
-              selectedValue={urlType}
+              selectedValue={ingestor.type}
-              onSelect={(value: { label: string; value: string }) =>
+              onSelect={(selected: { label: string; value: string }) =>
-                setUrlType(value)
+                handleIngestorTypeChange(selected.value as IngestorType)
               }
               size="w-full"
               rounded="3xl"
             />
-            {urlType.label !== 'Reddit' && urlType.label !== 'GitHub' ? (
+            {ingestor.type === 'reddit' ? (
               <>
                 <Input
-                  placeholder={`Enter ${t('modals.uploadDoc.name')}`}
+                  placeholder="Client ID"
                   type="text"
-                  value={urlName}
+                  name="client_id"
-                  onChange={(e) => setUrlName(e.target.value)}
+                  value={(ingestor.config as RedditIngestorConfig).client_id}
+                  onChange={handleIngestorChange}
                   borderVariant="thin"
-                ></Input>
+                />
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.name')}
-                  </span>
-                </div>
                 <Input
-                  placeholder={t('modals.uploadDoc.urlLink')}
+                  placeholder="Client Secret"
                   type="text"
-                  value={url}
+                  name="client_secret"
-                  onChange={(e) => setUrl(e.target.value)}
+                  value={
+                    (ingestor.config as RedditIngestorConfig).client_secret
+                  }
+                  onChange={handleIngestorChange}
                   borderVariant="thin"
-                ></Input>
+                />
-                <div className="relative bottom-12 left-2 mt-[-20px]">
+                <Input
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                  placeholder="User Agent"
-                    {t('modals.uploadDoc.link')}
+                  type="text"
-                  </span>
+                  name="user_agent"
-                </div>
+                  value={(ingestor.config as RedditIngestorConfig).user_agent}
-              </>
+                  onChange={handleIngestorChange}
-            ) : urlType.label === 'GitHub' ? ( // P3f93
+                  borderVariant="thin"
-              <>
+                />
-                <Input
+                <Input
-                  placeholder={`Enter ${t('modals.uploadDoc.name')}`}
+                  placeholder="Search Queries"
-                  type="text"
+                  type="text"
-                  value={urlName}
+                  name="search_queries"
-                  onChange={(e) => setUrlName(e.target.value)}
+                  value={
-                  borderVariant="thin"
+                    (ingestor.config as RedditIngestorConfig).search_queries
-                ></Input>
+                  }
-                <div className="relative bottom-12 left-2 mt-[-20px]">
+                  onChange={handleIngestorChange}
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                  borderVariant="thin"
-                    {t('modals.uploadDoc.name')}
+                />
-                  </span>
+                <Input
-                </div>
+                  placeholder="Number of Posts"
-                <Input
+                  type="number"
-                  placeholder={t('modals.uploadDoc.repoUrl')}
+                  name="number_posts"
-                  type="text"
+                  value={(ingestor.config as RedditIngestorConfig).number_posts}
-                  value={repoUrl}
+                  onChange={handleIngestorChange}
-                  onChange={(e) => setRepoUrl(e.target.value)}
+                  borderVariant="thin"
-                  borderVariant="thin"
+                />
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.repoUrl')}
-                  </span>
-                </div>
               </>
+            ) : ingestor.type === 'github' ? (
+              <Input
+                placeholder="Repository URL"
+                type="text"
+                name="repo_url"
+                value={(ingestor.config as GithubIngestorConfig).repo_url}
+                onChange={handleIngestorChange}
+                borderVariant="thin"
+              />
             ) : (
-              <div className="flex flex-col gap-1 mt-2">
+              <>
-                <div>
+                <Input
-                  <Input
+                  placeholder={`Enter ${t('modals.uploadDoc.name')}`}
-                    placeholder={t('modals.uploadDoc.reddit.id')}
+                  type="text"
-                    type="text"
+                  name="name"
-                    name="client_id"
+                  value={ingestor.name}
-                    value={redditData.client_id}
+                  onChange={(e) =>
-                    onChange={handleChange}
+                    setIngestor({ ...ingestor, name: e.target.value })
-                    borderVariant="thin"
+                  }
-                  ></Input>
+                  borderVariant="thin"
-                  <div className="relative bottom-[52px] left-2">
+                />
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                <Input
-                      {t('modals.uploadDoc.reddit.id')}
+                  placeholder="Enter URL"
-                    </span>
+                  type="text"
-                  </div>
+                  name="url"
-                </div>
+                  value={
-                <div>
+                    (
-                  <Input
+                      ingestor.config as
-                    placeholder={t('modals.uploadDoc.reddit.secret')}
+                        | CrawlerIngestorConfig
-                    type="text"
+                        | UrlIngestorConfig
-                    name="client_secret"
+                    ).url
-                    value={redditData.client_secret}
+                  }
-                    onChange={handleChange}
+                  onChange={handleIngestorChange}
-                    borderVariant="thin"
+                  borderVariant="thin"
-                  ></Input>
+                />
-                  <div className="relative bottom-[52px] left-2">
+              </>
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                      {t('modals.uploadDoc.reddit.secret')}
-                    </span>
-                  </div>
-                </div>
-                <div>
-                  <Input
-                    placeholder={t('modals.uploadDoc.reddit.agent')}
-                    type="text"
-                    name="user_agent"
-                    value={redditData.user_agent}
-                    onChange={handleChange}
-                    borderVariant="thin"
-                  ></Input>
-                  <div className="relative bottom-[52px] left-2">
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                      {t('modals.uploadDoc.reddit.agent')}
-                    </span>
-                  </div>
-                </div>
-                <div>
-                  <Input
-                    placeholder={t('modals.uploadDoc.reddit.searchQueries')}
-                    type="text"
-                    name="search_queries"
-                    value={redditData.search_queries}
-                    onChange={handleChange}
-                    borderVariant="thin"
-                  ></Input>
-                  <div className="relative bottom-[52px] left-2">
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                      {t('modals.uploadDoc.reddit.searchQueries')}
-                    </span>
-                  </div>
-                </div>
-                <div>
-                  <Input
-                    placeholder={t('modals.uploadDoc.reddit.numberOfPosts')}
-                    type="number"
-                    name="number_posts"
-                    value={redditData.number_posts}
-                    onChange={handleChange}
-                    borderVariant="thin"
-                  ></Input>
-                  <div className="relative bottom-[52px] left-2">
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                      {t('modals.uploadDoc.reddit.numberOfPosts')}
-                    </span>
-                  </div>
-                </div>
-              </div>
             )}
           </>
         )}
@@ -615,33 +722,9 @@ function Upload({
                   uploadRemote();
                 }
               }}
-              disabled={
+              disabled={isUploadDisabled()}
-                (activeTab === 'file' && (!files.length || !docName)) ||
-                (activeTab === 'remote' &&
-                  ((urlType.label !== 'Reddit' &&
-                    urlType.label !== 'GitHub' &&
-                    (!url || !urlName)) ||
-                    (urlType.label === 'GitHub' && !repoUrl) ||
-                    (urlType.label === 'Reddit' &&
-                      (!redditData.client_id ||
-                        !redditData.client_secret ||
-                        !redditData.user_agent ||
-                        !redditData.search_queries ||
-                        !redditData.number_posts))))
-              }
               className={`rounded-3xl px-4 py-2 font-medium ${
-                (activeTab === 'file' && (!files.length || !docName)) ||
+                isUploadDisabled()
-                (activeTab === 'remote' &&
-                  ((urlType.label !== 'Reddit' &&
-                    urlType.label !== 'GitHub' &&
-                    (!url || !urlName)) ||
-                    (urlType.label === 'GitHub' && !repoUrl) ||
-                    (urlType.label === 'Reddit' &&
-                      (!redditData.client_id ||
-                        !redditData.client_secret ||
-                        !redditData.user_agent ||
-                        !redditData.search_queries ||
-                        !redditData.number_posts))))
                   ? 'cursor-not-allowed bg-gray-300 text-gray-500'
                   : 'cursor-pointer bg-purple-30 text-white hover:bg-purple-40'
               }`}

frontend/src/upload/types/ingestor.ts

@@ -0,0 +1,43 @@
+export interface BaseIngestorConfig {
+  name: string;
+}
+
+export interface RedditIngestorConfig extends BaseIngestorConfig {
+  client_id: string;
+  client_secret: string;
+  user_agent: string;
+  search_queries: string[];
+  number_posts: number;
+}
+
+export interface GithubIngestorConfig extends BaseIngestorConfig {
+  repo_url: string;
+}
+
+export interface CrawlerIngestorConfig extends BaseIngestorConfig {
+  url: string;
+}
+
+export interface UrlIngestorConfig extends BaseIngestorConfig {
+  url: string;
+}
+
+export type IngestorType = 'crawler' | 'github' | 'reddit' | 'url';
+
+export interface IngestorConfig {
+  type: IngestorType;
+  name: string;
+  config:
+    | RedditIngestorConfig
+    | GithubIngestorConfig
+    | CrawlerIngestorConfig
+    | UrlIngestorConfig
+    | string;
+}
+
+export type IngestorFormData = {
+  name: string;
+  user: string;
+  source: IngestorType;
+  data: string;
+};