Merge pull request #1610 from ManishMadan2882/main

Refactor: Ingestor types for remote resources in Upload Component

frontend/src/components/Input.tsx

@@ -7,6 +7,7 @@ const Input = ({
   value,
   isAutoFocused = false,
   placeholder,
+  label,
   maxLength,
   className,
   colorVariant = 'silver',
@@ -26,21 +27,30 @@ const Input = ({
     thick: 'border-2',
   };
   return (
-    <input
-      className={`h-[42px] w-full rounded-full px-3 py-1 outline-none dark:bg-transparent dark:text-white ${className} ${colorStyles[colorVariant]} ${borderStyles[borderVariant]}`}
-      type={type}
-      id={id}
-      name={name}
-      autoFocus={isAutoFocused}
-      placeholder={placeholder}
-      maxLength={maxLength}
-      value={value}
-      onChange={onChange}
-      onPaste={onPaste}
-      onKeyDown={onKeyDown}
-    >
-      {children}
-    </input>
+    <div className="relative">
+      <input
+        className={`h-[42px] w-full rounded-full px-3 py-1 outline-none dark:bg-transparent dark:text-white ${className} ${colorStyles[colorVariant]} ${borderStyles[borderVariant]}`}
+        type={type}
+        id={id}
+        name={name}
+        autoFocus={isAutoFocused}
+        placeholder={placeholder}
+        maxLength={maxLength}
+        value={value}
+        onChange={onChange}
+        onPaste={onPaste}
+        onKeyDown={onKeyDown}
+      >
+        {children}
+      </input>
+      {label && (
+        <div className="absolute -top-2 left-2">
+          <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+            {label}
+          </span>
+        </div>
+      )}
+    </div>
   );
 };
 

frontend/src/components/ToggleSwitch.tsx

@@ -0,0 +1,58 @@
+import React from 'react';
+
+type ToggleSwitchProps = {
+  checked: boolean;
+  onChange: (checked: boolean) => void;
+  className?: string;
+  label?: string;
+  disabled?: boolean;
+  activeColor?: string;
+  inactiveColor?: string;
+  id?: string;
+};
+
+const ToggleSwitch: React.FC<ToggleSwitchProps> = ({
+  checked,
+  onChange,
+  className = '',
+  label,
+  disabled = false,
+  activeColor = 'bg-purple-30',
+  inactiveColor = 'bg-transparent',
+  id,
+}) => {
+  return (
+    <label
+      className={`cursor-pointer select-none justify-between flex flex-row items-center ${disabled ? 'opacity-50 cursor-not-allowed' : ''} ${className}`}
+      htmlFor={id}
+    >
+      {label && (
+        <span className="mr-2 text-eerie-black dark:text-white">{label}</span>
+      )}
+      <div className="relative">
+        <input
+          type="checkbox"
+          checked={checked}
+          onChange={(e) => onChange(e.target.checked)}
+          className="sr-only"
+          disabled={disabled}
+          id={id}
+        />
+        <div
+          className={`box block h-8 w-14 rounded-full border border-purple-30 ${
+            checked
+              ? `${activeColor} dark:${activeColor}`
+              : `${inactiveColor} dark:${inactiveColor}`
+          }`}
+        ></div>
+        <div
+          className={`absolute left-1 top-1 flex h-6 w-6 items-center justify-center rounded-full transition ${
+            checked ? 'translate-x-full bg-silver' : 'bg-purple-30'
+          }`}
+        ></div>
+      </div>
+    </label>
+  );
+};
+
+export default ToggleSwitch;

frontend/src/components/types/index.ts

@@ -8,6 +8,7 @@ export type InputProps = {
   maxLength?: number;
   name?: string;
   placeholder?: string;
+  label?: string;
   className?: string;
   children?: React.ReactElement;
   onChange: (

frontend/src/upload/Upload.tsx

@@ -8,6 +8,7 @@ import FileUpload from '../assets/file_upload.svg';
 import WebsiteCollect from '../assets/website_collect.svg';
 import Dropdown from '../components/Dropdown';
 import Input from '../components/Input';
+import ToggleSwitch from '../components/ToggleSwitch';
 import { ActiveState, Doc } from '../models/misc';
 import { getDocs } from '../preferences/preferenceApi';
 import {
@@ -16,6 +17,27 @@ import {
   selectSourceDocs,
 } from '../preferences/preferenceSlice';
 import WrapperModal from '../modals/WrapperModal';
+import {
+  IngestorType,
+  IngestorConfig,
+  RedditIngestorConfig,
+  GithubIngestorConfig,
+  CrawlerIngestorConfig,
+  UrlIngestorConfig,
+  IngestorFormSchemas,
+  FormField,
+} from './types/ingestor';
+import { IngestorDefaultConfigs } from '../upload/types/ingestor';
+
+type IngestorState = {
+  type: IngestorType;
+  name: string;
+  config:
+    | RedditIngestorConfig
+    | GithubIngestorConfig
+    | CrawlerIngestorConfig
+    | UrlIngestorConfig;
+};
 
 function Upload({
   receivedFile = [],
@@ -33,18 +55,106 @@ function Upload({
   onSuccessfulUpload?: () => void;
 }) {
   const [docName, setDocName] = useState(receivedFile[0]?.name);
-  const [urlName, setUrlName] = useState('');
-  const [url, setUrl] = useState('');
-  const [repoUrl, setRepoUrl] = useState(''); // P3f93
-  const [redditData, setRedditData] = useState({
-    client_id: '',
-    client_secret: '',
-    user_agent: '',
-    search_queries: [''],
-    number_posts: 10,
-  });
-  const [activeTab, setActiveTab] = useState<string | null>(renderTab);
   const [files, setfiles] = useState<File[]>(receivedFile);
+  const [activeTab, setActiveTab] = useState<string | null>(renderTab);
+
+  const renderFormFields = () => {
+    const schema = IngestorFormSchemas[ingestor.type];
+
+    return schema.map((field: FormField) => {
+      switch (field.type) {
+        case 'string':
+          return (
+            <div key={field.name} className="mb-4">
+              <Input
+                placeholder={field.label}
+                type="text"
+                name={field.name}
+                value={(ingestor.config as any)[field.name]}
+                onChange={(e) =>
+                  handleIngestorChange(field.name, e.target.value)
+                }
+                borderVariant="thin"
+                label={field.label}
+                colorVariant="gray"
+              />
+            </div>
+          );
+        case 'number':
+          return (
+            <div key={field.name} className="mb-4">
+              <Input
+                placeholder={field.label}
+                type="number"
+                name={field.name}
+                value={(ingestor.config as any)[field.name]}
+                onChange={(e) =>
+                  handleIngestorChange(field.name, parseInt(e.target.value))
+                }
+                borderVariant="thin"
+                label={field.label}
+                colorVariant="gray"
+              />
+            </div>
+          );
+        case 'enum':
+          return (
+            <div key={field.name} className="mb-4">
+              <Dropdown
+                key={field.name}
+                options={field.options || []}
+                selectedValue={(ingestor.config as any)[field.name]}
+                onSelect={(
+                  selected: { label: string; value: string } | string,
+                ) => {
+                  const value =
+                    typeof selected === 'string' ? selected : selected.value;
+                  handleIngestorChange(field.name, value);
+                }}
+                size="w-full"
+                rounded="3xl"
+                placeholder={field.label}
+                border="border"
+                borderColor="gray-5000"
+              />
+            </div>
+          );
+        case 'boolean':
+          return (
+            <div key={field.name} className="mb-4">
+              <ToggleSwitch
+                label={field.label}
+                checked={(ingestor.config as any)[field.name]}
+                onChange={(checked: boolean) => {
+                  const syntheticEvent = {
+                    target: {
+                      name: field.name,
+                      value: checked,
+                    },
+                  } as unknown as React.ChangeEvent<HTMLInputElement>;
+                  handleIngestorChange(field.name, syntheticEvent.target.value);
+                }}
+                className="mt-2"
+              />
+            </div>
+          );
+        default:
+          return null;
+      }
+    });
+  };
+
+  // New unified ingestor state
+  const [ingestor, setIngestor] = useState<IngestorConfig>(() => {
+    const defaultType: IngestorType = 'crawler';
+    const defaultConfig = IngestorDefaultConfigs[defaultType];
+    return {
+      type: defaultType,
+      name: defaultConfig.name,
+      config: defaultConfig.config,
+    };
+  });
+
   const [progress, setProgress] = useState<{
     type: 'UPLOAD' | 'TRAINING';
     percentage: number;
@@ -55,12 +165,11 @@ function Upload({
   const { t } = useTranslation();
   const setTimeoutRef = useRef<number | null>();
 
-  const urlOptions: { label: string; value: string }[] = [
-    { label: `Crawler`, value: 'crawler' },
-    // { label: t('modals.uploadDoc.sitemap'), value: 'sitemap' },
-    { label: `Link`, value: 'url' },
-    { label: `GitHub`, value: 'github' },
-    { label: `Reddit`, value: 'reddit' },
+  const urlOptions: { label: string; value: IngestorType }[] = [
+    { label: 'Crawler', value: 'crawler' },
+    { label: 'Link', value: 'url' },
+    { label: 'GitHub', value: 'github' },
+    { label: 'Reddit', value: 'reddit' },
   ];
 
   const [urlType, setUrlType] = useState<{ label: string; value: string }>({
@@ -264,7 +373,8 @@ function Upload({
     files.forEach((file) => {
       formData.append('file', file);
     });
-    formData.append('name', docName);
+
+    formData.append('name', activeTab === 'file' ? docName : ingestor.name);
     formData.append('user', 'local');
     const apiHost = import.meta.env.VITE_API_HOST;
     const xhr = new XMLHttpRequest();
@@ -284,22 +394,27 @@ function Upload({
 
   const uploadRemote = () => {
     const formData = new FormData();
-    formData.append('name', urlName);
+    formData.append('name', ingestor.name);
     formData.append('user', 'local');
-    if (urlType !== null) {
-      formData.append('source', urlType?.value);
-    }
-    formData.append('data', url);
-    if (
-      redditData.client_id.length > 0 &&
-      redditData.client_secret.length > 0
-    ) {
-      formData.set('name', 'other');
-      formData.set('data', JSON.stringify(redditData));
-    }
-    if (urlType.value === 'github') {
-      formData.append('repo_url', repoUrl); // Pdeac
+    formData.append('source', ingestor.type);
+
+    if (ingestor.type === 'reddit') {
+      const redditConfig = ingestor.config as RedditIngestorConfig;
+      redditConfig.name = ingestor.name;
+      formData.set('data', JSON.stringify(redditConfig));
+    } else if (ingestor.type === 'github') {
+      const githubConfig = ingestor.config as GithubIngestorConfig;
+      githubConfig.name = ingestor.name;
+      formData.append('repo_url', githubConfig.repo_url);
+      formData.append('data', githubConfig.repo_url);
+    } else {
+      const urlBasedConfig = ingestor.config as
+        | CrawlerIngestorConfig
+        | UrlIngestorConfig;
+      urlBasedConfig.name = ingestor.name;
+      formData.append('data', urlBasedConfig.url);
     }
+
     const apiHost = import.meta.env.VITE_API_HOST;
     const xhr = new XMLHttpRequest();
     xhr.upload.addEventListener('progress', (event) => {
@@ -346,20 +461,50 @@ function Upload({
     },
   });
 
-  const handleChange = (
-    e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>,
-  ) => {
-    const { name, value } = e.target;
-    if (name === 'search_queries' && value.length > 0) {
-      setRedditData({
-        ...redditData,
-        [name]: value.split(',').map((item) => item.trim()),
-      });
-    } else
-      setRedditData({
-        ...redditData,
-        [name]: name === 'number_posts' ? parseInt(value) : value,
-      });
+  const isUploadDisabled = () => {
+    if (activeTab === 'file') {
+      return !docName || files.length === 0;
+    }
+
+    if (activeTab !== 'remote') return false;
+
+    if (!ingestor.name) return true;
+
+    return Object.values(ingestor.config).some((value) => {
+      if (Array.isArray(value)) {
+        return value.length === 0;
+      }
+      return !value;
+    });
+  };
+
+  const handleIngestorChange = (key: string, value: any) => {
+    setIngestor((prevState: IngestorConfig): IngestorConfig => {
+      if (key === 'name') {
+        return {
+          ...prevState,
+          name: value,
+        };
+      }
+
+      return {
+        ...prevState,
+        config: {
+          ...(prevState.config as any),
+          [key]: value,
+        },
+      };
+    });
+  };
+
+  const handleIngestorTypeChange = (type: IngestorType) => {
+    const defaultConfig = IngestorDefaultConfigs[type];
+
+    setIngestor({
+      type,
+      name: defaultConfig.name,
+      config: defaultConfig.config,
+    });
   };
 
   let view;
@@ -455,146 +600,29 @@ function Upload({
             <Dropdown
               border="border"
               options={urlOptions}
-              selectedValue={urlType}
-              onSelect={(value: { label: string; value: string }) =>
-                setUrlType(value)
+              selectedValue={
+                urlOptions.find((opt) => opt.value === ingestor.type) || null
+              }
+              onSelect={(selected: { label: string; value: string }) =>
+                handleIngestorTypeChange(selected.value as IngestorType)
               }
               size="w-full"
               rounded="3xl"
             />
-            {urlType.label !== 'Reddit' && urlType.label !== 'GitHub' ? (
-              <>
-                <Input
-                  placeholder={`Enter ${t('modals.uploadDoc.name')}`}
-                  type="text"
-                  value={urlName}
-                  onChange={(e) => setUrlName(e.target.value)}
-                  borderVariant="thin"
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.name')}
-                  </span>
-                </div>
-                <Input
-                  placeholder={t('modals.uploadDoc.urlLink')}
-                  type="text"
-                  value={url}
-                  onChange={(e) => setUrl(e.target.value)}
-                  borderVariant="thin"
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.link')}
-                  </span>
-                </div>
-              </>
-            ) : urlType.label === 'GitHub' ? ( // P3f93
-              <>
-                <Input
-                  placeholder={`Enter ${t('modals.uploadDoc.name')}`}
-                  type="text"
-                  value={urlName}
-                  onChange={(e) => setUrlName(e.target.value)}
-                  borderVariant="thin"
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.name')}
-                  </span>
-                </div>
-                <Input
-                  placeholder={t('modals.uploadDoc.repoUrl')}
-                  type="text"
-                  value={repoUrl}
-                  onChange={(e) => setRepoUrl(e.target.value)}
-                  borderVariant="thin"
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.repoUrl')}
-                  </span>
-                </div>
-              </>
-            ) : (
-              <div className="flex flex-col gap-1 mt-2">
-                <div>
-                  <Input
-                    placeholder={t('modals.uploadDoc.reddit.id')}
-                    type="text"
-                    name="client_id"
-                    value={redditData.client_id}
-                    onChange={handleChange}
-                    borderVariant="thin"
-                  ></Input>
-                  <div className="relative bottom-[52px] left-2">
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                      {t('modals.uploadDoc.reddit.id')}
-                    </span>
-                  </div>
-                </div>
-                <div>
-                  <Input
-                    placeholder={t('modals.uploadDoc.reddit.secret')}
-                    type="text"
-                    name="client_secret"
-                    value={redditData.client_secret}
-                    onChange={handleChange}
-                    borderVariant="thin"
-                  ></Input>
-                  <div className="relative bottom-[52px] left-2">
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                      {t('modals.uploadDoc.reddit.secret')}
-                    </span>
-                  </div>
-                </div>
-                <div>
-                  <Input
-                    placeholder={t('modals.uploadDoc.reddit.agent')}
-                    type="text"
-                    name="user_agent"
-                    value={redditData.user_agent}
-                    onChange={handleChange}
-                    borderVariant="thin"
-                  ></Input>
-                  <div className="relative bottom-[52px] left-2">
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                      {t('modals.uploadDoc.reddit.agent')}
-                    </span>
-                  </div>
-                </div>
-                <div>
-                  <Input
-                    placeholder={t('modals.uploadDoc.reddit.searchQueries')}
-                    type="text"
-                    name="search_queries"
-                    value={redditData.search_queries}
-                    onChange={handleChange}
-                    borderVariant="thin"
-                  ></Input>
-                  <div className="relative bottom-[52px] left-2">
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                      {t('modals.uploadDoc.reddit.searchQueries')}
-                    </span>
-                  </div>
-                </div>
-                <div>
-                  <Input
-                    placeholder={t('modals.uploadDoc.reddit.numberOfPosts')}
-                    type="number"
-                    name="number_posts"
-                    value={redditData.number_posts}
-                    onChange={handleChange}
-                    borderVariant="thin"
-                  ></Input>
-                  <div className="relative bottom-[52px] left-2">
-                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                      {t('modals.uploadDoc.reddit.numberOfPosts')}
-                    </span>
-                  </div>
-                </div>
-              </div>
-            )}
+            {/* Dynamically render form fields based on schema */}
+
+            <Input
+              type="text"
+              colorVariant="gray"
+              value={ingestor['name']}
+              onChange={(e) =>
+                setIngestor({ ...ingestor, name: e.target.value })
+              }
+              borderVariant="thin"
+              placeholder="Name"
+              label="Name"
+            />
+            {renderFormFields()}
           </>
         )}
         <div className="flex justify-between">
@@ -615,33 +643,8 @@ function Upload({
                   uploadRemote();
                 }
               }}
-              disabled={
-                (activeTab === 'file' && (!files.length || !docName)) ||
-                (activeTab === 'remote' &&
-                  ((urlType.label !== 'Reddit' &&
-                    urlType.label !== 'GitHub' &&
-                    (!url || !urlName)) ||
-                    (urlType.label === 'GitHub' && !repoUrl) ||
-                    (urlType.label === 'Reddit' &&
-                      (!redditData.client_id ||
-                        !redditData.client_secret ||
-                        !redditData.user_agent ||
-                        !redditData.search_queries ||
-                        !redditData.number_posts))))
-              }
               className={`rounded-3xl px-4 py-2 font-medium ${
-                (activeTab === 'file' && (!files.length || !docName)) ||
-                (activeTab === 'remote' &&
-                  ((urlType.label !== 'Reddit' &&
-                    urlType.label !== 'GitHub' &&
-                    (!url || !urlName)) ||
-                    (urlType.label === 'GitHub' && !repoUrl) ||
-                    (urlType.label === 'Reddit' &&
-                      (!redditData.client_id ||
-                        !redditData.client_secret ||
-                        !redditData.user_agent ||
-                        !redditData.search_queries ||
-                        !redditData.number_posts))))
+                isUploadDisabled()
                   ? 'cursor-not-allowed bg-gray-300 text-gray-500'
                   : 'cursor-pointer bg-purple-30 text-white hover:bg-purple-40'
               }`}

frontend/src/upload/types/ingestor.ts

@@ -0,0 +1,136 @@
+export interface BaseIngestorConfig {
+  name: string;
+}
+
+export interface RedditIngestorConfig extends BaseIngestorConfig {
+  client_id: string;
+  client_secret: string;
+  user_agent: string;
+  search_queries: string;
+  number_posts: number;
+}
+
+export interface GithubIngestorConfig extends BaseIngestorConfig {
+  repo_url: string;
+}
+
+export interface CrawlerIngestorConfig extends BaseIngestorConfig {
+  url: string;
+}
+
+export interface UrlIngestorConfig extends BaseIngestorConfig {
+  url: string;
+}
+
+export type IngestorType = 'crawler' | 'github' | 'reddit' | 'url';
+
+export interface IngestorConfig {
+  type: IngestorType;
+  name: string;
+  config:
+    | RedditIngestorConfig
+    | GithubIngestorConfig
+    | CrawlerIngestorConfig
+    | UrlIngestorConfig;
+}
+
+export type IngestorFormData = {
+  name: string;
+  user: string;
+  source: IngestorType;
+  data: string;
+};
+
+export type FieldType = 'string' | 'number' | 'enum' | 'boolean';
+
+export interface FormField {
+  name: keyof BaseIngestorConfig | string;
+  label: string;
+  type: FieldType;
+  options?: { label: string; value: string }[];
+}
+
+export const IngestorFormSchemas: Record<IngestorType, FormField[]> = {
+  crawler: [
+    {
+      name: 'url',
+      label: 'URL',
+      type: 'string',
+    },
+  ],
+  url: [
+    {
+      name: 'url',
+      label: 'URL',
+      type: 'string',
+    },
+  ],
+  reddit: [
+    {
+      name: 'client_id',
+      label: 'Client ID',
+      type: 'string',
+    },
+    {
+      name: 'client_secret',
+      label: 'Client Secret',
+      type: 'string',
+    },
+    {
+      name: 'user_agent',
+      label: 'User Agent',
+      type: 'string',
+    },
+    {
+      name: 'search_queries',
+      label: 'Search Queries',
+      type: 'string',
+    },
+    {
+      name: 'number_posts',
+      label: 'Number of Posts',
+      type: 'number',
+    },
+  ],
+  github: [
+    {
+      name: 'repo_url',
+      label: 'Repository URL',
+      type: 'string',
+    },
+  ],
+};
+
+export const IngestorDefaultConfigs: Record<
+  IngestorType,
+  Omit<IngestorConfig, 'type'>
+> = {
+  crawler: {
+    name: '',
+    config: {
+      url: '',
+    } as CrawlerIngestorConfig,
+  },
+  url: {
+    name: '',
+    config: {
+      url: '',
+    } as UrlIngestorConfig,
+  },
+  reddit: {
+    name: '',
+    config: {
+      client_id: '',
+      client_secret: '',
+      user_agent: '',
+      search_queries: '',
+      number_posts: 10,
+    } as RedditIngestorConfig,
+  },
+  github: {
+    name: '',
+    config: {
+      repo_url: '',
+    } as GithubIngestorConfig,
+  },
+};