import { useCallback, useEffect, useRef, useState } from 'react'; import { useDropzone } from 'react-dropzone'; import { useTranslation } from 'react-i18next'; import { useDispatch, useSelector } from 'react-redux'; import userService from '../api/services/userService'; import { getSessionToken } from '../utils/providerUtils'; import Dropdown from '../components/Dropdown'; import Input from '../components/Input'; import ToggleSwitch from '../components/ToggleSwitch'; import WrapperModal from '../modals/WrapperModal'; import { ActiveState, Doc } from '../models/misc'; import { getDocs } from '../preferences/preferenceApi'; import { selectSourceDocs, selectToken, setSelectedDocs, setSourceDocs, } from '../preferences/preferenceSlice'; import { IngestorDefaultConfigs, IngestorFormSchemas, getIngestorSchema, IngestorOption, } from '../upload/types/ingestor'; import { FormField, IngestorConfig, IngestorType } from './types/ingestor'; import { FilePicker } from '../components/FilePicker'; import GoogleDrivePicker from '../components/GoogleDrivePicker'; import ChevronRight from '../assets/chevron-right.svg'; function Upload({ receivedFile = [], setModalState, isOnboarding, renderTab = null, close, onSuccessfulUpload = () => undefined, }: { receivedFile: File[]; setModalState: (state: ActiveState) => void; isOnboarding: boolean; renderTab: string | null; close: () => void; onSuccessfulUpload?: () => void; }) { const token = useSelector(selectToken); const [files, setfiles] = useState(receivedFile); const [activeTab, setActiveTab] = useState(true); const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); // File picker state const [selectedFiles, setSelectedFiles] = useState([]); const [selectedFolders, setSelectedFolders] = useState([]); const renderFormFields = () => { if (!ingestor.type) return null; const ingestorSchema = getIngestorSchema(ingestor.type as IngestorType); if (!ingestorSchema) return null; const schema: FormField[] = ingestorSchema.fields; const generalFields = schema.filter((field: FormField) => !field.advanced); const advancedFields = schema.filter((field: FormField) => field.advanced); return (
{generalFields.map((field: FormField) => renderField(field))}
{advancedFields.length > 0 && (

{advancedFields.map((field: FormField) => renderField(field))}
)}
); }; const renderField = (field: FormField) => { const isRequired = field.required ?? false; switch (field.type) { case 'string': return ( handleIngestorChange( field.name as keyof IngestorConfig['config'], e.target.value, ) } borderVariant="thin" required={isRequired} colorVariant="silver" labelBgClassName="bg-white dark:bg-charleston-green-2" /> ); case 'number': return ( handleIngestorChange( field.name as keyof IngestorConfig['config'], Number(e.target.value), ) } borderVariant="thin" required={isRequired} colorVariant="silver" labelBgClassName="bg-white dark:bg-charleston-green-2" /> ); case 'enum': return ( opt.value === ingestor.config[field.name as keyof typeof ingestor.config], ) || null } onSelect={(selected: { label: string; value: string }) => { handleIngestorChange( field.name as keyof IngestorConfig['config'], selected.value, ); }} size="w-full" rounded="3xl" placeholder={field.label} border="border" buttonClassName="border-silver bg-white dark:border-dim-gray dark:bg-[#222327]" optionsClassName="border-silver bg-white dark:border-dim-gray dark:bg-[#383838]" placeholderClassName="text-gray-400 dark:text-silver" contentSize="text-sm" /> ); case 'boolean': return ( { handleIngestorChange( field.name as keyof IngestorConfig['config'], checked, ); }} size="small" className={`mt-2 text-base`} /> ); case 'local_file_picker': return (
Choose Files

Selected Files

{files.map((file) => (

{file.name}

))} {files.length === 0 && (

No files selected

)}
); case 'remote_file_picker': return ( { setSelectedFiles(selectedFileIds); setSelectedFolders(selectedFolderIds); }} provider={ingestor.type as unknown as string} token={token} initialSelectedFiles={selectedFiles} initialSelectedFolders={selectedFolders} /> ); case 'google_drive_picker': return ( { setSelectedFiles(selectedFileIds); setSelectedFolders(selectedFolderIds); }} token={token} /> ); default: return null; } }; // New unified ingestor state const [ingestor, setIngestor] = useState(() => ({ type: null, name: '', config: {}, })); const [progress, setProgress] = useState<{ type: 'UPLOAD' | 'TRAINING'; percentage: number; taskId?: string; failed?: boolean; }>(); const { t } = useTranslation(); const setTimeoutRef = useRef(null); const ingestorOptions: IngestorOption[] = IngestorFormSchemas.filter( (schema) => (schema.validate ? schema.validate() : true), ).map((schema) => ({ label: schema.label, value: schema.key, icon: schema.icon, heading: schema.heading, })); const sourceDocs = useSelector(selectSourceDocs); useEffect(() => { if (setTimeoutRef.current) { clearTimeout(setTimeoutRef.current); } }, []); function ProgressBar({ progressPercent }: { progressPercent: number }) { return (
{progressPercent}%
); } function Progress({ title, isCancellable = false, isFailed = false, isTraining = false, }: { title: string; isCancellable?: boolean; isFailed?: boolean; isTraining?: boolean; }) { return (

{isTraining && (progress?.percentage === 100 ? t('modals.uploadDoc.progress.completed') : title)} {!isTraining && title}

{t('modals.uploadDoc.progress.wait')}

{t('modals.uploadDoc.progress.tokenLimit')}

{/*

{progress?.percentage || 0}%

*/} {isTraining && (progress?.percentage === 100 ? ( ) : ( ))}
); } function UploadProgress() { return ; } function TrainingProgress() { const dispatch = useDispatch(); useEffect(() => { let timeoutID: number | undefined; if ((progress?.percentage ?? 0) < 100) { timeoutID = setTimeout(() => { userService .getTaskStatus(progress?.taskId as string, null) .then((data) => data.json()) .then((data) => { if (data.status == 'SUCCESS') { if (data.result.limited === true) { getDocs(token).then((data) => { dispatch(setSourceDocs(data)); dispatch( setSelectedDocs( Array.isArray(data) && data?.find( (d: Doc) => d.type?.toLowerCase() === 'local', ), ), ); }); setProgress( (progress) => progress && { ...progress, percentage: 100, failed: true, }, ); } else { getDocs(token).then((data) => { dispatch(setSourceDocs(data)); const docIds = new Set( (Array.isArray(sourceDocs) && sourceDocs?.map((doc: Doc) => doc.id ? doc.id : null, )) || [], ); if (data && Array.isArray(data)) { data.map((updatedDoc: Doc) => { if (updatedDoc.id && !docIds.has(updatedDoc.id)) { // Select the doc not present in the intersection of current Docs and fetched data dispatch(setSelectedDocs(updatedDoc)); return; } }); } }); setProgress( (progress) => progress && { ...progress, percentage: 100, failed: false, }, ); setIngestor({ type: null, name: '', config: {} }); setfiles([]); setProgress(undefined); setModalState('INACTIVE'); onSuccessfulUpload?.(); } } else if (data.status == 'PROGRESS') { setProgress( (progress) => progress && { ...progress, percentage: data.result.current, }, ); } }); }, 5000); } // cleanup return () => { if (timeoutID !== undefined) { clearTimeout(timeoutID); } }; }, [progress, dispatch]); return ( ); } const onDrop = useCallback( (acceptedFiles: File[]) => { setfiles(acceptedFiles); setIngestor((prev) => ({ ...prev, name: acceptedFiles[0]?.name || '' })); // If we're in local_file mode, update the ingestor config if (ingestor.type === 'local_file') { setIngestor((prevState) => ({ ...prevState, config: { ...prevState.config, files: acceptedFiles, }, })); } }, [ingestor.type], ); const doNothing = () => undefined; const uploadFile = () => { const formData = new FormData(); files.forEach((file) => { formData.append('file', file); }); formData.append('name', ingestor.name); formData.append('user', 'local'); const apiHost = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); xhr.upload.addEventListener('progress', (event) => { const progress = +((event.loaded / event.total) * 100).toFixed(2); setProgress({ type: 'UPLOAD', percentage: progress }); }); xhr.onload = () => { const { task_id } = JSON.parse(xhr.responseText); setTimeoutRef.current = setTimeout(() => { setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id }); }, 3000); }; xhr.open('POST', `${apiHost + '/api/upload'}`); xhr.setRequestHeader('Authorization', `Bearer ${token}`); xhr.send(formData); }; const uploadRemote = () => { if (!ingestor.type) return; const formData = new FormData(); formData.append('name', ingestor.name); formData.append('user', 'local'); formData.append('source', ingestor.type as string); let configData: any = {}; const ingestorSchema = getIngestorSchema(ingestor.type as IngestorType); if (!ingestorSchema) return; const schema: FormField[] = ingestorSchema.fields; const hasLocalFilePicker = schema.some( (field: FormField) => field.type === 'local_file_picker', ); const hasRemoteFilePicker = schema.some( (field: FormField) => field.type === 'remote_file_picker', ); const hasGoogleDrivePicker = schema.some( (field: FormField) => field.type === 'google_drive_picker', ); if (hasLocalFilePicker) { files.forEach((file) => { formData.append('file', file); }); configData = { ...ingestor.config }; } else if (hasRemoteFilePicker || hasGoogleDrivePicker) { const sessionToken = getSessionToken(ingestor.type as string); configData = { provider: ingestor.type as string, session_token: sessionToken, file_ids: selectedFiles, folder_ids: selectedFolders, }; } else { configData = { ...ingestor.config }; } formData.append('data', JSON.stringify(configData)); const apiHost: string = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); xhr.upload.addEventListener('progress', (event: ProgressEvent) => { if (event.lengthComputable) { const progressPercentage = +( (event.loaded / event.total) * 100 ).toFixed(2); setProgress({ type: 'UPLOAD', percentage: progressPercentage }); } }); xhr.onload = () => { const response = JSON.parse(xhr.responseText) as { task_id: string }; setTimeoutRef.current = window.setTimeout(() => { setProgress({ type: 'TRAINING', percentage: 0, taskId: response.task_id, }); }, 3000); }; const endpoint = ingestor.type === 'local_file' ? `${apiHost}/api/upload` : `${apiHost}/api/remote`; xhr.open('POST', endpoint); xhr.setRequestHeader('Authorization', `Bearer ${token}`); xhr.send(formData); }; const { getRootProps, getInputProps } = useDropzone({ onDrop, multiple: true, onDragEnter: doNothing, onDragOver: doNothing, onDragLeave: doNothing, maxSize: 25000000, accept: { 'application/pdf': ['.pdf'], 'text/plain': ['.txt'], 'text/x-rst': ['.rst'], 'text/x-markdown': ['.md'], 'application/zip': ['.zip'], 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'], 'application/json': ['.json'], 'text/csv': ['.csv'], 'text/html': ['.html'], 'application/epub+zip': ['.epub'], 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': [ '.xlsx', ], 'application/vnd.openxmlformats-officedocument.presentationml.presentation': ['.pptx'], 'image/png': ['.png'], 'image/jpeg': ['.jpeg'], 'image/jpg': ['.jpg'], }, }); const isUploadDisabled = (): boolean => { if (!activeTab) return true; if (!ingestor.name?.trim()) { return true; } if (!ingestor.type) return true; const ingestorSchemaForValidation = getIngestorSchema( ingestor.type as IngestorType, ); if (!ingestorSchemaForValidation) return true; const schema: FormField[] = ingestorSchemaForValidation.fields; const hasLocalFilePicker = schema.some( (field: FormField) => field.type === 'local_file_picker', ); const hasRemoteFilePicker = schema.some( (field: FormField) => field.type === 'remote_file_picker', ); const hasGoogleDrivePicker = schema.some( (field: FormField) => field.type === 'google_drive_picker', ); if (hasLocalFilePicker) { if (files.length === 0) { return true; } } else if (hasRemoteFilePicker || hasGoogleDrivePicker) { if (selectedFiles.length === 0 && selectedFolders.length === 0) { return true; } } const ingestorSchemaForFields = getIngestorSchema( ingestor.type as IngestorType, ); if (!ingestorSchemaForFields) return false; const formFields: FormField[] = ingestorSchemaForFields.fields; for (const field of formFields) { if (field.required) { // Validate only required fields const value = ingestor.config[field.name as keyof typeof ingestor.config]; if (typeof value === 'string' && !value.trim()) { return true; } if ( typeof value === 'number' && (value === null || value === undefined || value <= 0) ) { return true; } if (typeof value === 'boolean' && value === undefined) { return true; } } } return false; }; const handleIngestorChange = ( key: keyof IngestorConfig['config'], value: string | number | boolean, ) => { setIngestor((prevState) => ({ ...prevState, config: { ...prevState.config, [key]: value, }, })); }; const handleIngestorTypeChange = (type: IngestorType | null) => { if (type === null) { setIngestor({ type: null, name: '', config: {}, }); setfiles([]); return; } const defaultConfig = IngestorDefaultConfigs[type]; setIngestor({ type, name: defaultConfig.name, config: defaultConfig.config, }); // Clear files if switching away from local_file if (type !== 'local_file') { setfiles([]); } }; const renderIngestorSelection = () => { return (
{ingestorOptions.map((option) => (
handleIngestorTypeChange(option.value as IngestorType) } >
{option.label}

{option.label}

))}
); }; let view; if (progress?.type === 'UPLOAD') { view = ; } else if (progress?.type === 'TRAINING') { view = ; } else { view = (
{!ingestor.type && (

Select the way to add your source

)} {activeTab && ( <> {!ingestor.type && renderIngestorSelection()} {ingestor.type && (

{ingestor.type && getIngestorSchema(ingestor.type as IngestorType)?.heading}

{ setIngestor((prevState) => ({ ...prevState, name: e.target.value, })); }} borderVariant="thin" placeholder="Name" required={true} labelBgClassName="bg-white dark:bg-charleston-green-2" className="w-full" /> {renderFormFields()}
)} {ingestor.type && getIngestorSchema(ingestor.type as IngestorType)?.fields.some( (field: FormField) => field.advanced, ) && ( )} )}
{activeTab && ingestor.type && ( )}
); } return ( { close(); setIngestor({ type: null, name: '', config: {} }); setfiles([]); setModalState('INACTIVE'); }} className="max-h-[90vh] w-11/12 sm:max-h-none sm:w-auto sm:min-w-[600px] md:min-w-[700px]" contentClassName="max-h-[80vh] sm:max-h-none" > {view} ); } export default Upload;