import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useDropzone } from 'react-dropzone'; import { useTranslation } from 'react-i18next'; import { useDispatch, useSelector } from 'react-redux'; import userService from '../api/services/userService'; import { getSessionToken, setSessionToken, removeSessionToken } from '../utils/providerUtils'; import { formatDate } from '../utils/dateTimeUtils'; import { formatBytes } from '../utils/stringUtils'; import FileUpload from '../assets/file_upload.svg'; import WebsiteCollect from '../assets/website_collect.svg'; import Dropdown from '../components/Dropdown'; import Input from '../components/Input'; import ToggleSwitch from '../components/ToggleSwitch'; import WrapperModal from '../modals/WrapperModal'; import { ActiveState, Doc } from '../models/misc'; import { getDocs } from '../preferences/preferenceApi'; import { selectSourceDocs, selectToken, setSelectedDocs, setSourceDocs, } from '../preferences/preferenceSlice'; import { IngestorDefaultConfigs } from '../upload/types/ingestor'; import { FormField, IngestorConfig, IngestorFormSchemas, IngestorType, } from './types/ingestor'; import FileIcon from '../assets/file.svg'; import FolderIcon from '../assets/folder.svg'; import ConnectorAuth from '../components/ConnectorAuth'; function Upload({ receivedFile = [], setModalState, isOnboarding, renderTab = null, close, onSuccessfulUpload = () => undefined, }: { receivedFile: File[]; setModalState: (state: ActiveState) => void; isOnboarding: boolean; renderTab: string | null; close: () => void; onSuccessfulUpload?: () => void; }) { const token = useSelector(selectToken); const [docName, setDocName] = useState(receivedFile[0]?.name); const [remoteName, setRemoteName] = useState(''); const [files, setfiles] = useState(receivedFile); const [activeTab, setActiveTab] = useState(renderTab); const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); // Google Drive state const [isGoogleDriveConnected, setIsGoogleDriveConnected] = useState(false); const [googleDriveFiles, setGoogleDriveFiles] = useState([]); const [selectedFiles, setSelectedFiles] = useState([]); const [isLoadingFiles, setIsLoadingFiles] = useState(false); const [isAuthenticating, setIsAuthenticating] = useState(false); const [userEmail, setUserEmail] = useState(''); const [authError, setAuthError] = useState(''); const [currentFolderId, setCurrentFolderId] = useState(null); const [folderPath, setFolderPath] = useState>([{id: null, name: 'My Drive'}]); const [nextPageToken, setNextPageToken] = useState(null); const [hasMoreFiles, setHasMoreFiles] = useState(false); const scrollContainerRef = useRef(null); const renderFormFields = () => { const schema = IngestorFormSchemas[ingestor.type]; if (!schema) return null; const generalFields = schema.filter((field) => !field.advanced); const advancedFields = schema.filter((field) => field.advanced); return (
{generalFields.map((field: FormField) => renderField(field))}
{advancedFields.length > 0 && (

{advancedFields.map((field: FormField) => renderField(field))}
)}
); }; const renderField = (field: FormField) => { const isRequired = field.required ?? false; switch (field.type) { case 'string': return ( handleIngestorChange( field.name as keyof IngestorConfig['config'], e.target.value, ) } borderVariant="thin" required={isRequired} colorVariant="silver" labelBgClassName="bg-white dark:bg-charleston-green-2" /> ); case 'number': return ( handleIngestorChange( field.name as keyof IngestorConfig['config'], Number(e.target.value), ) } borderVariant="thin" required={isRequired} colorVariant="silver" labelBgClassName="bg-white dark:bg-charleston-green-2" /> ); case 'enum': return ( opt.value === ingestor.config[field.name as keyof typeof ingestor.config], ) || null } onSelect={(selected: { label: string; value: string }) => { handleIngestorChange( field.name as keyof IngestorConfig['config'], selected.value, ); }} size="w-full" rounded="3xl" placeholder={field.label} border="border" buttonClassName="border-silver bg-white dark:border-dim-gray dark:bg-[#222327]" optionsClassName="border-silver bg-white dark:border-dim-gray dark:bg-[#383838]" placeholderClassName="text-gray-400 dark:text-silver" contentSize="text-sm" /> ); case 'boolean': return ( { handleIngestorChange( field.name as keyof IngestorConfig['config'], checked, ); }} className="mt-2" /> ); default: return null; } }; // New unified ingestor state const [ingestor, setIngestor] = useState(() => { const defaultType: IngestorType = 'crawler'; const defaultConfig = IngestorDefaultConfigs[defaultType]; return { type: defaultType, name: defaultConfig.name, config: defaultConfig.config, }; }); const [progress, setProgress] = useState<{ type: 'UPLOAD' | 'TRAINING'; percentage: number; taskId?: string; failed?: boolean; }>(); const { t } = useTranslation(); const setTimeoutRef = useRef(null); const urlOptions: { label: string; value: IngestorType }[] = [ { label: 'Crawler', value: 'crawler' }, { label: 'Link', value: 'url' }, { label: 'GitHub', value: 'github' }, { label: 'Reddit', value: 'reddit' }, { label: 'Google Drive', value: 'google_drive' }, ]; const sourceDocs = useSelector(selectSourceDocs); useEffect(() => { if (setTimeoutRef.current) { clearTimeout(setTimeoutRef.current); } }, []); function ProgressBar({ progressPercent }: { progressPercent: number }) { return (
{progressPercent}%
); } function Progress({ title, isCancellable = false, isFailed = false, isTraining = false, }: { title: string; isCancellable?: boolean; isFailed?: boolean; isTraining?: boolean; }) { return (

{isTraining && (progress?.percentage === 100 ? t('modals.uploadDoc.progress.completed') : title)} {!isTraining && title}

{t('modals.uploadDoc.progress.wait')}

{t('modals.uploadDoc.progress.tokenLimit')}

{/*

{progress?.percentage || 0}%

*/} {isTraining && (progress?.percentage === 100 ? ( ) : ( ))}
); } function UploadProgress() { return ; } function TrainingProgress() { const dispatch = useDispatch(); useEffect(() => { let timeoutID: number | undefined; if ((progress?.percentage ?? 0) < 100) { timeoutID = setTimeout(() => { userService .getTaskStatus(progress?.taskId as string, null) .then((data) => data.json()) .then((data) => { if (data.status == 'SUCCESS') { if (data.result.limited === true) { getDocs(token).then((data) => { dispatch(setSourceDocs(data)); dispatch( setSelectedDocs( Array.isArray(data) && data?.find( (d: Doc) => d.type?.toLowerCase() === 'local', ), )); }); setProgress( (progress) => progress && { ...progress, percentage: 100, failed: true, }, ); } else { getDocs(token).then((data) => { dispatch(setSourceDocs(data)); const docIds = new Set( (Array.isArray(sourceDocs) && sourceDocs?.map((doc: Doc) => doc.id ? doc.id : null, )) || [], ); if (data && Array.isArray(data)) { data.map((updatedDoc: Doc) => { if (updatedDoc.id && !docIds.has(updatedDoc.id)) { // Select the doc not present in the intersection of current Docs and fetched data dispatch(setSelectedDocs(updatedDoc)); return; } }); } }); setProgress( (progress) => progress && { ...progress, percentage: 100, failed: false, }, ); setDocName(''); setfiles([]); setProgress(undefined); setModalState('INACTIVE'); onSuccessfulUpload?.(); } } else if (data.status == 'PROGRESS') { setProgress( (progress) => progress && { ...progress, percentage: data.result.current, }, ); } }); }, 5000); } // cleanup return () => { if (timeoutID !== undefined) { clearTimeout(timeoutID); } }; }, [progress, dispatch]); return ( ); } const onDrop = useCallback((acceptedFiles: File[]) => { setfiles(acceptedFiles); setDocName(acceptedFiles[0]?.name || ''); }, []); const doNothing = () => undefined; const uploadFile = () => { const formData = new FormData(); files.forEach((file) => { formData.append('file', file); }); formData.append('name', docName); formData.append('user', 'local'); const apiHost = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); xhr.upload.addEventListener('progress', (event) => { const progress = +((event.loaded / event.total) * 100).toFixed(2); setProgress({ type: 'UPLOAD', percentage: progress }); }); xhr.onload = () => { const { task_id } = JSON.parse(xhr.responseText); setTimeoutRef.current = setTimeout(() => { setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id }); }, 3000); }; xhr.open('POST', `${apiHost + '/api/upload'}`); xhr.setRequestHeader('Authorization', `Bearer ${token}`); xhr.send(formData); }; const uploadRemote = () => { const formData = new FormData(); formData.append('name', remoteName); formData.append('user', 'local'); formData.append('source', ingestor.type); let configData; if (ingestor.type === 'google_drive') { const sessionToken = getSessionToken(ingestor.type); const selectedItems = googleDriveFiles.filter(file => selectedFiles.includes(file.id)); const selectedFolderIds = selectedItems .filter(item => item.type === 'application/vnd.google-apps.folder' || item.isFolder) .map(folder => folder.id); const selectedFileIds = selectedItems .filter(item => item.type !== 'application/vnd.google-apps.folder' && !item.isFolder) .map(file => file.id); configData = { file_ids: selectedFileIds, folder_ids: selectedFolderIds, recursive: ingestor.config.recursive, session_token: sessionToken || null }; } else { configData = { ...ingestor.config }; } formData.append('data', JSON.stringify(configData)); const apiHost: string = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); xhr.upload.addEventListener('progress', (event: ProgressEvent) => { if (event.lengthComputable) { const progressPercentage = +( (event.loaded / event.total) * 100 ).toFixed(2); setProgress({ type: 'UPLOAD', percentage: progressPercentage }); } }); xhr.onload = () => { const response = JSON.parse(xhr.responseText) as { task_id: string }; setTimeoutRef.current = window.setTimeout(() => { setProgress({ type: 'TRAINING', percentage: 0, taskId: response.task_id, }); }, 3000); }; xhr.open('POST', `${apiHost}/api/remote`); xhr.setRequestHeader('Authorization', `Bearer ${token}`); xhr.send(formData); }; useEffect(() => { if (ingestor.type === 'google_drive') { const sessionToken = getSessionToken(ingestor.type); if (sessionToken) { // Auto-authenticate if session token exists setIsGoogleDriveConnected(true); setAuthError(''); // Fetch user email and files using the existing session token fetchUserEmailAndLoadFiles(sessionToken); } } }, [ingestor.type]); const fetchUserEmailAndLoadFiles = async (sessionToken: string) => { try { const apiHost = import.meta.env.VITE_API_HOST; const validateResponse = await fetch(`${apiHost}/api/connectors/validate-session`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${token}` }, body: JSON.stringify({ provider: 'google_drive', session_token: sessionToken }) }); if (!validateResponse.ok) { removeSessionToken(ingestor.type); setIsGoogleDriveConnected(false); setAuthError('Session expired. Please reconnect to Google Drive.'); return; } const validateData = await validateResponse.json(); if (validateData.success) { setUserEmail(validateData.user_email || 'Connected User'); // reset pagination state and files setGoogleDriveFiles([]); setNextPageToken(null); setHasMoreFiles(false); loadGoogleDriveFiles(sessionToken, null, null, false); } else { removeSessionToken(ingestor.type); setIsGoogleDriveConnected(false); setAuthError(validateData.error || 'Session expired. Please reconnect your Google Drive account and make sure to grant offline access.'); } } catch (error) { console.error('Error validating Google Drive session:', error); setAuthError('Failed to validate session. Please reconnect.'); setIsGoogleDriveConnected(false); } }; const loadGoogleDriveFiles = async ( sessionToken: string, folderId?: string | null, pageToken?: string | null, append: boolean = false, ) => { setIsLoadingFiles(true); try { const apiHost = import.meta.env.VITE_API_HOST; const requestBody: any = { session_token: sessionToken, limit: 10, }; if (folderId) { requestBody.folder_id = folderId; } if (pageToken) { requestBody.page_token = pageToken; } const filesResponse = await fetch(`${apiHost}/api/connectors/files`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${token}` }, body: JSON.stringify({ ...requestBody, provider: 'google_drive' }) }); if (!filesResponse.ok) { throw new Error(`Failed to load files: ${filesResponse.status}`); } const filesData = await filesResponse.json(); if (filesData.success && Array.isArray(filesData.files)) { setGoogleDriveFiles(prev => append ? [...prev, ...filesData.files] : filesData.files); setNextPageToken(filesData.next_page_token || null); setHasMoreFiles(Boolean(filesData.has_more)); } else { throw new Error(filesData.error || 'Failed to load files'); } } catch (error) { console.error('Error loading Google Drive files:', error); setAuthError(error instanceof Error ? error.message : 'Failed to load files. Please make sure your Google Drive account is properly connected and you granted offline access during authorization.'); } finally { setIsLoadingFiles(false); } }; // Handle file selection const handleFileSelect = (fileId: string) => { setSelectedFiles(prev => { if (prev.includes(fileId)) { return prev.filter(id => id !== fileId); } else { return [...prev, fileId]; } }); }; const handleFolderClick = (folderId: string, folderName: string) => { const sessionToken = getSessionToken(ingestor.type); if (sessionToken) { setCurrentFolderId(folderId); setFolderPath(prev => [...prev, {id: folderId, name: folderName}]); setGoogleDriveFiles([]); setNextPageToken(null); setHasMoreFiles(false); setSelectedFiles([]); loadGoogleDriveFiles(sessionToken, folderId, null, false); } }; const navigateBack = (index: number) => { const sessionToken = getSessionToken(ingestor.type); if (sessionToken) { const newPath = folderPath.slice(0, index + 1); const targetFolderId = newPath[newPath.length - 1]?.id; setCurrentFolderId(targetFolderId as string | null); setFolderPath(newPath); setGoogleDriveFiles([]); setNextPageToken(null); setHasMoreFiles(false); setSelectedFiles([]); loadGoogleDriveFiles(sessionToken, targetFolderId ?? null, null, false); } }; const handleSelectAll = () => { if (selectedFiles.length === googleDriveFiles.length) { setSelectedFiles([]); } else { setSelectedFiles(googleDriveFiles.map(file => file.id)); } }; const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, multiple: true, onDragEnter: doNothing, onDragOver: doNothing, onDragLeave: doNothing, maxSize: 25000000, accept: { 'application/pdf': ['.pdf'], 'text/plain': ['.txt'], 'text/x-rst': ['.rst'], 'text/x-markdown': ['.md'], 'application/zip': ['.zip'], 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'], 'application/json': ['.json'], 'text/csv': ['.csv'], 'text/html': ['.html'], 'application/epub+zip': ['.epub'], 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': [ '.xlsx', ], 'application/vnd.openxmlformats-officedocument.presentationml.presentation': ['.pptx'], 'image/png': ['.png'], 'image/jpeg': ['.jpeg'], 'image/jpg': ['.jpg'], }, }); const isUploadDisabled = (): boolean => { if (activeTab === 'file') { return !docName?.trim() || files.length === 0; } if (activeTab === 'remote') { if (!remoteName?.trim()) { return true; } if (ingestor.type === 'google_drive') { return !isGoogleDriveConnected || selectedFiles.length === 0; } const formFields: FormField[] = IngestorFormSchemas[ingestor.type]; for (const field of formFields) { if (field.required) { // Validate only required fields const value = ingestor.config[field.name as keyof typeof ingestor.config]; if (typeof value === 'string' && !value.trim()) { return true; } if ( typeof value === 'number' && (value === null || value === undefined || value <= 0) ) { return true; } if (typeof value === 'boolean' && value === undefined) { return true; } } } return false; } return true; }; const handleIngestorChange = ( key: keyof IngestorConfig['config'], value: string | number | boolean, ) => { setIngestor((prevState) => ({ ...prevState, config: { ...prevState.config, [key]: value, }, })); }; const handleIngestorTypeChange = (type: IngestorType) => { //Updates the ingestor seleced in dropdown and resets the config to the default config for that type const defaultConfig = IngestorDefaultConfigs[type]; setIngestor({ type, name: defaultConfig.name, config: defaultConfig.config, }); }; let view; if (progress?.type === 'UPLOAD') { view = ; } else if (progress?.type === 'TRAINING') { view = ; } else { view = (

{t('modals.uploadDoc.label')}

{!activeTab && (

{t('modals.uploadDoc.select')}

)} {activeTab === 'file' && ( <> setDocName(e.target.value)} borderVariant="thin" placeholder={t('modals.uploadDoc.name')} labelBgClassName="bg-white dark:bg-charleston-green-2" required={true} />
{t('modals.uploadDoc.choose')}

{t('modals.uploadDoc.info')}

{t('modals.uploadDoc.uploadedFiles')}

{files.map((file) => (

{file.name}

))} {files.length === 0 && (

{t('none')}

)}
)} {activeTab === 'remote' && ( <> opt.value === ingestor.type) || null } onSelect={(selected: { label: string; value: string }) => handleIngestorTypeChange(selected.value as IngestorType) } size="w-full" rounded="3xl" border="border" placeholder="Select ingestor type" placeholderClassName="text-gray-400 dark:text-silver" /> {/* Dynamically render form fields based on schema */} setRemoteName(e.target.value)} borderVariant="thin" placeholder="Name" required={true} labelBgClassName="bg-white dark:bg-charleston-green-2" /> {ingestor.type === 'google_drive' && (
{authError && (

⚠️ {authError}

)} {!isGoogleDriveConnected ? ( { setUserEmail(data.user_email); setIsGoogleDriveConnected(true); setIsAuthenticating(false); setAuthError(''); if (data.session_token) { setSessionToken(ingestor.type, data.session_token); loadGoogleDriveFiles(data.session_token, null); } }} onError={(error) => { setAuthError(error); setIsAuthenticating(false); setIsGoogleDriveConnected(false); }} /> ) : (
{/* Connection Status */}
Connected as {userEmail}
{/* File Browser */}
{/* Breadcrumb navigation */}
{folderPath.map((path, index) => (
{index > 0 && /}
))}

Select Files from Google Drive

{googleDriveFiles.length > 0 && ( )}
{selectedFiles.length > 0 && (

{selectedFiles.length} file{selectedFiles.length !== 1 ? 's' : ''} selected

)}
{isLoadingFiles && googleDriveFiles.length === 0 ? (
Loading files...
) : googleDriveFiles.length === 0 ? (
No files found in your Google Drive
) : ( <>
{googleDriveFiles.map((file) => (
handleFileSelect(file.id)} className="h-4 w-4 text-blue-600 rounded border-gray-300 focus:ring-blue-500" />
{file.type === 'application/vnd.google-apps.folder' || file.isFolder ? (
handleFolderClick(file.id, file.name)} > Folder
) : (
File
)}

{ if (file.type === 'application/vnd.google-apps.folder' || file.isFolder) { handleFolderClick(file.id, file.name); } }} > {file.name}

{file.size && `${formatBytes(file.size)} • `}Modified {formatDate(file.modifiedTime)}

))}
{isLoadingFiles && (
Loading more files...
)} {!hasMoreFiles && !isLoadingFiles && ( All files loaded )}
)}
)}
)} {renderFormFields()} {IngestorFormSchemas[ingestor.type].some( (field) => field.advanced, ) && ( )} )}
{activeTab && ( )} {activeTab && ( )}
); } useEffect(() => { const scrollContainer = scrollContainerRef.current; const handleScroll = () => { if (!scrollContainer) return; const { scrollTop, scrollHeight, clientHeight } = scrollContainer; const isNearBottom = scrollHeight - scrollTop - clientHeight < 50; if (isNearBottom && hasMoreFiles && !isLoadingFiles && nextPageToken) { const sessionToken = getSessionToken(ingestor.type); if (sessionToken) { loadGoogleDriveFiles(sessionToken, currentFolderId, nextPageToken, true); } } }; scrollContainer?.addEventListener('scroll', handleScroll); return () => { scrollContainer?.removeEventListener('scroll', handleScroll); }; }, [hasMoreFiles, isLoadingFiles, nextPageToken, currentFolderId, ingestor.type]); return ( { close(); setDocName(''); setfiles([]); setModalState('INACTIVE'); setActiveTab(null); }} > {view} ); } export default Upload;