import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useDropzone } from 'react-dropzone'; import { useTranslation } from 'react-i18next'; import { useDispatch, useSelector } from 'react-redux'; import userService from '../api/services/userService'; import FileUpload from '../assets/file_upload.svg'; import WebsiteCollect from '../assets/website_collect.svg'; import Dropdown from '../components/Dropdown'; import Input from '../components/Input'; import ToggleSwitch from '../components/ToggleSwitch'; import { ActiveState, Doc } from '../models/misc'; import { getDocs } from '../preferences/preferenceApi'; import { setSelectedDocs, setSourceDocs, selectSourceDocs, } from '../preferences/preferenceSlice'; import WrapperModal from '../modals/WrapperModal'; import { IngestorType, IngestorConfig, IngestorFormSchemas, FormField, } from './types/ingestor'; import { IngestorDefaultConfigs } from '../upload/types/ingestor'; function Upload({ receivedFile = [], setModalState, isOnboarding, renderTab = null, close, onSuccessfulUpload = () => undefined, }: { receivedFile: File[]; setModalState: (state: ActiveState) => void; isOnboarding: boolean; renderTab: string | null; close: () => void; onSuccessfulUpload?: () => void; }) { const [docName, setDocName] = useState(receivedFile[0]?.name); const [remoteName, setRemoteName] = useState(''); const [files, setfiles] = useState(receivedFile); const [activeTab, setActiveTab] = useState(renderTab); const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); const renderFormFields = () => { const schema = IngestorFormSchemas[ingestor.type]; if (!schema) return null; const generalFields = schema.filter((field) => !field.advanced); const advancedFields = schema.filter((field) => field.advanced); return (
{generalFields.map((field: FormField) => renderField(field))}
{advancedFields.length > 0 && (

{advancedFields.map((field: FormField) => renderField(field))}
)}
); }; const renderField = (field: FormField) => { const isRequired = field.required ?? false; switch (field.type) { case 'string': return ( handleIngestorChange( field.name as keyof IngestorConfig['config'], e.target.value, ) } borderVariant="thin" label={field.label} required={isRequired} colorVariant="gray" /> ); case 'number': return ( handleIngestorChange( field.name as keyof IngestorConfig['config'], Number(e.target.value), ) } borderVariant="thin" label={field.label} required={isRequired} colorVariant="gray" /> ); case 'enum': return ( opt.value === ingestor.config[field.name as keyof typeof ingestor.config], ) || null } onSelect={(selected: { label: string; value: string }) => { handleIngestorChange( field.name as keyof IngestorConfig['config'], selected.value, ); }} size="w-full" rounded="3xl" placeholder={field.label} border="border" borderColor="gray-5000" /> ); case 'boolean': return ( { handleIngestorChange( field.name as keyof IngestorConfig['config'], checked, ); }} className="mt-2" /> ); default: return null; } }; // New unified ingestor state const [ingestor, setIngestor] = useState(() => { const defaultType: IngestorType = 'crawler'; const defaultConfig = IngestorDefaultConfigs[defaultType]; return { type: defaultType, name: defaultConfig.name, config: defaultConfig.config, }; }); const [progress, setProgress] = useState<{ type: 'UPLOAD' | 'TRAINING'; percentage: number; taskId?: string; failed?: boolean; }>(); const { t } = useTranslation(); const setTimeoutRef = useRef(); const urlOptions: { label: string; value: IngestorType }[] = [ { label: 'Crawler', value: 'crawler' }, { label: 'Link', value: 'url' }, { label: 'GitHub', value: 'github' }, { label: 'Reddit', value: 'reddit' }, ]; const sourceDocs = useSelector(selectSourceDocs); useEffect(() => { if (setTimeoutRef.current) { clearTimeout(setTimeoutRef.current); } }, []); function ProgressBar({ progressPercent }: { progressPercent: number }) { return (
{progressPercent}%
); } function Progress({ title, isCancellable = false, isFailed = false, isTraining = false, }: { title: string; isCancellable?: boolean; isFailed?: boolean; isTraining?: boolean; }) { return (

{isTraining && (progress?.percentage === 100 ? t('modals.uploadDoc.progress.completed') : title)} {!isTraining && title}

{t('modals.uploadDoc.progress.wait')}

{t('modals.uploadDoc.progress.tokenLimit')}

{/*

{progress?.percentage || 0}%

*/} {isTraining && (progress?.percentage === 100 ? ( ) : ( ))}
); } function UploadProgress() { return ; } function TrainingProgress() { const dispatch = useDispatch(); useEffect(() => { let timeoutID: number | undefined; if ((progress?.percentage ?? 0) < 100) { timeoutID = setTimeout(() => { userService .getTaskStatus(progress?.taskId as string) .then((data) => data.json()) .then((data) => { if (data.status == 'SUCCESS') { if (data.result.limited === true) { getDocs().then((data) => { dispatch(setSourceDocs(data)); dispatch( setSelectedDocs( Array.isArray(data) && data?.find( (d: Doc) => d.type?.toLowerCase() === 'local', ), ), ); }); setProgress( (progress) => progress && { ...progress, percentage: 100, failed: true, }, ); } else { getDocs().then((data) => { dispatch(setSourceDocs(data)); const docIds = new Set( (Array.isArray(sourceDocs) && sourceDocs?.map((doc: Doc) => doc.id ? doc.id : null, )) || [], ); if (data && Array.isArray(data)) { data.map((updatedDoc: Doc) => { if (updatedDoc.id && !docIds.has(updatedDoc.id)) { // Select the doc not present in the intersection of current Docs and fetched data dispatch(setSelectedDocs(updatedDoc)); return; } }); } }); setProgress( (progress) => progress && { ...progress, percentage: 100, failed: false, }, ); setDocName(''); setfiles([]); setProgress(undefined); setModalState('INACTIVE'); onSuccessfulUpload?.(); } } else if (data.status == 'PROGRESS') { setProgress( (progress) => progress && { ...progress, percentage: data.result.current, }, ); } }); }, 5000); } // cleanup return () => { if (timeoutID !== undefined) { clearTimeout(timeoutID); } }; }, [progress, dispatch]); return ( ); } const onDrop = useCallback((acceptedFiles: File[]) => { setfiles(acceptedFiles); setDocName(acceptedFiles[0]?.name || ''); }, []); const doNothing = () => undefined; const uploadFile = () => { const formData = new FormData(); files.forEach((file) => { formData.append('file', file); }); formData.append('name', docName); formData.append('user', 'local'); const apiHost = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); xhr.upload.addEventListener('progress', (event) => { const progress = +((event.loaded / event.total) * 100).toFixed(2); setProgress({ type: 'UPLOAD', percentage: progress }); }); xhr.onload = () => { const { task_id } = JSON.parse(xhr.responseText); setTimeoutRef.current = setTimeout(() => { setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id }); }, 3000); }; xhr.open('POST', `${apiHost + '/api/upload'}`); xhr.send(formData); }; const uploadRemote = () => { const formData = new FormData(); formData.append('name', remoteName); formData.append('user', 'local'); formData.append('source', ingestor.type); const defaultConfig = IngestorDefaultConfigs[ingestor.type].config; const mergedConfig = { ...defaultConfig, ...ingestor.config }; const filteredConfig = Object.entries(mergedConfig).reduce( (acc, [key, value]) => { const field = IngestorFormSchemas[ingestor.type].find( (f) => f.name === key, ); // Include the field if: // 1. It's required, or // 2. It's optional and has a non-empty value if ( field?.required || (value !== undefined && value !== null && value !== '') ) { acc[key] = value; } return acc; }, {} as Record, ); formData.append('data', JSON.stringify(filteredConfig)); const apiHost: string = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); xhr.upload.addEventListener('progress', (event: ProgressEvent) => { if (event.lengthComputable) { const progressPercentage = +( (event.loaded / event.total) * 100 ).toFixed(2); setProgress({ type: 'UPLOAD', percentage: progressPercentage }); } }); xhr.onload = () => { const response = JSON.parse(xhr.responseText) as { task_id: string }; setTimeoutRef.current = window.setTimeout(() => { setProgress({ type: 'TRAINING', percentage: 0, taskId: response.task_id, }); }, 3000); }; xhr.open('POST', `${apiHost}/api/remote`); xhr.send(formData); }; const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, multiple: true, onDragEnter: doNothing, onDragOver: doNothing, onDragLeave: doNothing, maxSize: 25000000, accept: { 'application/pdf': ['.pdf'], 'text/plain': ['.txt'], 'text/x-rst': ['.rst'], 'text/x-markdown': ['.md'], 'application/zip': ['.zip'], 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'], 'application/json': ['.json'], 'text/csv': ['.csv'], 'text/html': ['.html'], 'application/epub+zip': ['.epub'], 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': [ '.xlsx', ], 'application/vnd.openxmlformats-officedocument.presentationml.presentation': ['.pptx'], 'image/png': ['.png'], 'image/jpeg': ['.jpeg'], 'image/jpg': ['.jpg'], }, }); const isUploadDisabled = (): boolean => { if (activeTab === 'file') { return !docName?.trim() || files.length === 0; } if (activeTab === 'remote') { if (!remoteName?.trim()) { return true; } const formFields: FormField[] = IngestorFormSchemas[ingestor.type]; for (const field of formFields) { if (field.required) { // Validate only required fields const value = ingestor.config[field.name as keyof typeof ingestor.config]; if (typeof value === 'string' && !value.trim()) { return true; } if ( typeof value === 'number' && (value === null || value === undefined || value <= 0) ) { return true; } if (typeof value === 'boolean' && value === undefined) { return true; } } } return false; } return true; }; const handleIngestorChange = ( key: keyof IngestorConfig['config'], value: string | number | boolean, ) => { setIngestor((prevState) => ({ ...prevState, config: { ...prevState.config, [key]: value, }, })); }; const handleIngestorTypeChange = (type: IngestorType) => { //Updates the ingestor seleced in dropdown and resets the config to the default config for that type const defaultConfig = IngestorDefaultConfigs[type]; setIngestor({ type, name: defaultConfig.name, config: defaultConfig.config, }); }; let view; if (progress?.type === 'UPLOAD') { view = ; } else if (progress?.type === 'TRAINING') { view = ; } else { view = (

{t('modals.uploadDoc.label')}

{!activeTab && (

{t('modals.uploadDoc.select')}

)} {activeTab === 'file' && ( <> setDocName(e.target.value)} borderVariant="thin" >
{t('modals.uploadDoc.name')}
{t('modals.uploadDoc.choose')}

{t('modals.uploadDoc.info')}

{t('modals.uploadDoc.uploadedFiles')}

{files.map((file) => (

{file.name}

))} {files.length === 0 && (

{t('none')}

)}
)} {activeTab === 'remote' && ( <> opt.value === ingestor.type) || null } onSelect={(selected: { label: string; value: string }) => handleIngestorTypeChange(selected.value as IngestorType) } size="w-full" rounded="3xl" /> {/* Dynamically render form fields based on schema */} setRemoteName(e.target.value)} borderVariant="thin" placeholder="Name" label="Name" required={true} /> {renderFormFields()} {IngestorFormSchemas[ingestor.type].some( (field) => field.advanced, ) && ( )} )}
{activeTab && ( )} {activeTab && ( )}
); } return ( { close(); setDocName(''); setfiles([]); setModalState('INACTIVE'); setActiveTab(null); }} > {view} ); } export default Upload;