import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useDropzone } from 'react-dropzone'; import { useTranslation } from 'react-i18next'; import { useDispatch, useSelector } from 'react-redux'; import userService from '../api/services/userService'; import FileUpload from '../assets/file_upload.svg'; import WebsiteCollect from '../assets/website_collect.svg'; import Dropdown from '../components/Dropdown'; import Input from '../components/Input'; import ToggleSwitch from '../components/ToggleSwitch'; import WrapperModal from '../modals/WrapperModal'; import { ActiveState, Doc } from '../models/misc'; import { getDocs } from '../preferences/preferenceApi'; import { selectSourceDocs, selectToken, setSelectedDocs, setSourceDocs, } from '../preferences/preferenceSlice'; import { IngestorDefaultConfigs } from '../upload/types/ingestor'; import { FormField, IngestorConfig, IngestorFormSchemas, IngestorType, } from './types/ingestor'; import FileIcon from '../assets/file.svg'; import FolderIcon from '../assets/folder.svg'; function Upload({ receivedFile = [], setModalState, isOnboarding, renderTab = null, close, onSuccessfulUpload = () => undefined, }: { receivedFile: File[]; setModalState: (state: ActiveState) => void; isOnboarding: boolean; renderTab: string | null; close: () => void; onSuccessfulUpload?: () => void; }) { const token = useSelector(selectToken); const [docName, setDocName] = useState(receivedFile[0]?.name); const [remoteName, setRemoteName] = useState(''); const [files, setfiles] = useState(receivedFile); const [activeTab, setActiveTab] = useState(renderTab); const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); // Google Drive state const [isGoogleDriveConnected, setIsGoogleDriveConnected] = useState(false); const [googleDriveFiles, setGoogleDriveFiles] = useState([]); const [selectedFiles, setSelectedFiles] = useState([]); const [isLoadingFiles, setIsLoadingFiles] = useState(false); const [isAuthenticating, setIsAuthenticating] = useState(false); const [userEmail, setUserEmail] = useState(''); const [authError, setAuthError] = useState(''); const [currentFolderId, setCurrentFolderId] = useState(null); const [folderPath, setFolderPath] = useState>([{id: null, name: 'My Drive'}]); const renderFormFields = () => { const schema = IngestorFormSchemas[ingestor.type]; if (!schema) return null; const generalFields = schema.filter((field) => !field.advanced); const advancedFields = schema.filter((field) => field.advanced); return (
{generalFields.map((field: FormField) => renderField(field))}
{advancedFields.length > 0 && (

{advancedFields.map((field: FormField) => renderField(field))}
)}
); }; const renderField = (field: FormField) => { const isRequired = field.required ?? false; switch (field.type) { case 'string': return ( handleIngestorChange( field.name as keyof IngestorConfig['config'], e.target.value, ) } borderVariant="thin" required={isRequired} colorVariant="silver" labelBgClassName="bg-white dark:bg-charleston-green-2" /> ); case 'number': return ( handleIngestorChange( field.name as keyof IngestorConfig['config'], Number(e.target.value), ) } borderVariant="thin" required={isRequired} colorVariant="silver" labelBgClassName="bg-white dark:bg-charleston-green-2" /> ); case 'enum': return ( opt.value === ingestor.config[field.name as keyof typeof ingestor.config], ) || null } onSelect={(selected: { label: string; value: string }) => { handleIngestorChange( field.name as keyof IngestorConfig['config'], selected.value, ); }} size="w-full" rounded="3xl" placeholder={field.label} border="border" buttonClassName="border-silver bg-white dark:border-dim-gray dark:bg-[#222327]" optionsClassName="border-silver bg-white dark:border-dim-gray dark:bg-[#383838]" placeholderClassName="text-gray-400 dark:text-silver" contentSize="text-sm" /> ); case 'boolean': return ( { handleIngestorChange( field.name as keyof IngestorConfig['config'], checked, ); }} className="mt-2" /> ); default: return null; } }; // New unified ingestor state const [ingestor, setIngestor] = useState(() => { const defaultType: IngestorType = 'crawler'; const defaultConfig = IngestorDefaultConfigs[defaultType]; return { type: defaultType, name: defaultConfig.name, config: defaultConfig.config, }; }); const [progress, setProgress] = useState<{ type: 'UPLOAD' | 'TRAINING'; percentage: number; taskId?: string; failed?: boolean; }>(); const { t } = useTranslation(); const setTimeoutRef = useRef(null); const urlOptions: { label: string; value: IngestorType }[] = [ { label: 'Crawler', value: 'crawler' }, { label: 'Link', value: 'url' }, { label: 'GitHub', value: 'github' }, { label: 'Reddit', value: 'reddit' }, { label: 'Google Drive', value: 'google_drive' }, ]; const sourceDocs = useSelector(selectSourceDocs); useEffect(() => { if (setTimeoutRef.current) { clearTimeout(setTimeoutRef.current); } }, []); function ProgressBar({ progressPercent }: { progressPercent: number }) { return (
{progressPercent}%
); } function Progress({ title, isCancellable = false, isFailed = false, isTraining = false, }: { title: string; isCancellable?: boolean; isFailed?: boolean; isTraining?: boolean; }) { return (

{isTraining && (progress?.percentage === 100 ? t('modals.uploadDoc.progress.completed') : title)} {!isTraining && title}

{t('modals.uploadDoc.progress.wait')}

{t('modals.uploadDoc.progress.tokenLimit')}

{/*

{progress?.percentage || 0}%

*/} {isTraining && (progress?.percentage === 100 ? ( ) : ( ))}
); } function UploadProgress() { return ; } function TrainingProgress() { const dispatch = useDispatch(); useEffect(() => { let timeoutID: number | undefined; if ((progress?.percentage ?? 0) < 100) { timeoutID = setTimeout(() => { userService .getTaskStatus(progress?.taskId as string, null) .then((data) => data.json()) .then((data) => { if (data.status == 'SUCCESS') { if (data.result.limited === true) { getDocs(token).then((data) => { dispatch(setSourceDocs(data)); dispatch( setSelectedDocs( Array.isArray(data) && data?.find( (d: Doc) => d.type?.toLowerCase() === 'local', ), ), ); }); setProgress( (progress) => progress && { ...progress, percentage: 100, failed: true, }, ); } else { getDocs(token).then((data) => { dispatch(setSourceDocs(data)); const docIds = new Set( (Array.isArray(sourceDocs) && sourceDocs?.map((doc: Doc) => doc.id ? doc.id : null, )) || [], ); if (data && Array.isArray(data)) { data.map((updatedDoc: Doc) => { if (updatedDoc.id && !docIds.has(updatedDoc.id)) { // Select the doc not present in the intersection of current Docs and fetched data dispatch(setSelectedDocs(updatedDoc)); return; } }); } }); setProgress( (progress) => progress && { ...progress, percentage: 100, failed: false, }, ); setDocName(''); setfiles([]); setProgress(undefined); setModalState('INACTIVE'); onSuccessfulUpload?.(); } } else if (data.status == 'PROGRESS') { setProgress( (progress) => progress && { ...progress, percentage: data.result.current, }, ); } }); }, 5000); } // cleanup return () => { if (timeoutID !== undefined) { clearTimeout(timeoutID); } }; }, [progress, dispatch]); return ( ); } const onDrop = useCallback((acceptedFiles: File[]) => { setfiles(acceptedFiles); setDocName(acceptedFiles[0]?.name || ''); }, []); const doNothing = () => undefined; const uploadFile = () => { const formData = new FormData(); files.forEach((file) => { formData.append('file', file); }); formData.append('name', docName); formData.append('user', 'local'); const apiHost = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); xhr.upload.addEventListener('progress', (event) => { const progress = +((event.loaded / event.total) * 100).toFixed(2); setProgress({ type: 'UPLOAD', percentage: progress }); }); xhr.onload = () => { const { task_id } = JSON.parse(xhr.responseText); setTimeoutRef.current = setTimeout(() => { setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id }); }, 3000); }; xhr.open('POST', `${apiHost + '/api/upload'}`); xhr.setRequestHeader('Authorization', `Bearer ${token}`); xhr.send(formData); }; const uploadRemote = () => { const formData = new FormData(); formData.append('name', remoteName); formData.append('user', 'local'); formData.append('source', ingestor.type); let configData; if (ingestor.type === 'google_drive') { const sessionToken = localStorage.getItem('google_drive_session_token'); const selectedItems = googleDriveFiles.filter(file => selectedFiles.includes(file.id)); const selectedFolderIds = selectedItems .filter(item => item.type === 'application/vnd.google-apps.folder' || item.isFolder) .map(folder => folder.id); const selectedFileIds = selectedItems .filter(item => item.type !== 'application/vnd.google-apps.folder' && !item.isFolder) .map(file => file.id); configData = { file_ids: selectedFileIds, folder_ids: selectedFolderIds, recursive: ingestor.config.recursive, session_token: sessionToken || null }; } else { configData = { ...ingestor.config }; } formData.append('data', JSON.stringify(configData)); const apiHost: string = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); xhr.upload.addEventListener('progress', (event: ProgressEvent) => { if (event.lengthComputable) { const progressPercentage = +( (event.loaded / event.total) * 100 ).toFixed(2); setProgress({ type: 'UPLOAD', percentage: progressPercentage }); } }); xhr.onload = () => { const response = JSON.parse(xhr.responseText) as { task_id: string }; setTimeoutRef.current = window.setTimeout(() => { setProgress({ type: 'TRAINING', percentage: 0, taskId: response.task_id, }); }, 3000); }; xhr.open('POST', `${apiHost}/api/remote`); xhr.setRequestHeader('Authorization', `Bearer ${token}`); xhr.send(formData); }; useEffect(() => { if (ingestor.type === 'google_drive') { const sessionToken = localStorage.getItem('google_drive_session_token'); if (sessionToken) { // Auto-authenticate if session token exists setIsGoogleDriveConnected(true); setAuthError(''); // Fetch user email and files using the existing session token fetchUserEmailAndLoadFiles(sessionToken); } } }, [ingestor.type]); const fetchUserEmailAndLoadFiles = async (sessionToken: string) => { try { const apiHost = import.meta.env.VITE_API_HOST; const validateResponse = await fetch(`${apiHost}/api/google-drive/validate-session`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${token}` }, body: JSON.stringify({ session_token: sessionToken }) }); if (!validateResponse.ok) { localStorage.removeItem('google_drive_session_token'); setIsGoogleDriveConnected(false); setAuthError('Session expired. Please reconnect to Google Drive.'); return; } const validateData = await validateResponse.json(); if (validateData.success) { setUserEmail(validateData.user_email || 'Connected User'); loadGoogleDriveFiles(sessionToken, null); } else { localStorage.removeItem('google_drive_session_token'); setIsGoogleDriveConnected(false); setAuthError(validateData.error || 'Session expired. Please reconnect your Google Drive account and make sure to grant offline access.'); } } catch (error) { console.error('Error validating Google Drive session:', error); setAuthError('Failed to validate session. Please reconnect.'); setIsGoogleDriveConnected(false); } }; const handleGoogleDriveConnect = async () => { console.log('Google Drive connect button clicked'); setIsAuthenticating(true); setAuthError(''); const existingToken = localStorage.getItem('google_drive_session_token'); if (existingToken) { fetchUserEmailAndLoadFiles(existingToken); setIsAuthenticating(false); return; } try { const apiHost = import.meta.env.VITE_API_HOST; const authResponse = await fetch(`${apiHost}/api/google-drive/auth`, { headers: { 'Authorization': `Bearer ${token}` } }); if (!authResponse.ok) { throw new Error(`Failed to get authorization URL: ${authResponse.status}`); } const authData = await authResponse.json(); if (!authData.success || !authData.authorization_url) { throw new Error(authData.error || 'Failed to get authorization URL'); } console.log('Opening Google OAuth window...'); const authWindow = window.open( authData.authorization_url, 'google-drive-auth', 'width=500,height=600,scrollbars=yes,resizable=yes' ); if (!authWindow) { throw new Error('Failed to open authentication window. Please allow popups.'); } const handleAuthMessage = (event: MessageEvent) => { console.log('Received message event:', event.data); if (event.data.type === 'google_drive_auth_success') { console.log('OAuth success received:', event.data); setUserEmail(event.data.user_email || 'Connected User'); setIsGoogleDriveConnected(true); setIsAuthenticating(false); setAuthError(''); if (event.data.session_token) { localStorage.setItem('google_drive_session_token', event.data.session_token); } window.removeEventListener('message', handleAuthMessage); loadGoogleDriveFiles(event.data.session_token, null); } else if (event.data.type === 'google_drive_auth_error') { console.error('OAuth error received:', event.data); setAuthError(event.data.error || 'Authentication failed. Please make sure to grant all requested permissions, including offline access. You may need to revoke previous access and re-authorize.'); setIsAuthenticating(false); setIsGoogleDriveConnected(false); window.removeEventListener('message', handleAuthMessage); } }; window.addEventListener('message', handleAuthMessage); const checkClosed = setInterval(() => { if (authWindow.closed) { clearInterval(checkClosed); window.removeEventListener('message', handleAuthMessage); if (!isGoogleDriveConnected && !isAuthenticating) { setAuthError('Authentication was cancelled'); } } }, 1000); } catch (error) { console.error('Error during Google Drive authentication:', error); setAuthError(error instanceof Error ? error.message : 'Authentication failed'); setIsAuthenticating(false); } }; const loadGoogleDriveFiles = async (sessionToken: string, folderId?: string | null) => { setIsLoadingFiles(true); try { const apiHost = import.meta.env.VITE_API_HOST; const requestBody: any = { session_token: sessionToken, limit: 50 }; if (folderId) { requestBody.folder_id = folderId; } const filesResponse = await fetch(`${apiHost}/api/google-drive/files`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${token}` }, body: JSON.stringify(requestBody) }); if (!filesResponse.ok) { throw new Error(`Failed to load files: ${filesResponse.status}`); } const filesData = await filesResponse.json(); if (filesData.success && filesData.files) { setGoogleDriveFiles(filesData.files); } else { throw new Error(filesData.error || 'Failed to load files'); } } catch (error) { console.error('Error loading Google Drive files:', error); setAuthError(error instanceof Error ? error.message : 'Failed to load files. Please make sure your Google Drive account is properly connected and you granted offline access during authorization.'); // Fallback to mock data for demo purposes console.log('Using mock data as fallback...'); const mockFiles = [ { id: '1', name: 'Project Documentation.pdf', type: 'application/pdf', size: '2.5 MB', modifiedTime: '2024-01-15', iconUrl: '�' }, { id: '2', name: 'Meeting Notes.docx', type: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', size: '1.2 MB', modifiedTime: '2024-01-14', iconUrl: '�' }, { id: '3', name: 'Presentation.pptx', type: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', size: '5.8 MB', modifiedTime: '2024-01-13', iconUrl: '�' }, { id: 'folder1', name: 'Documents', type: 'application/vnd.google-apps.folder', size: '0 bytes', modifiedTime: '2024-01-13', iconUrl: '📁', isFolder: true } ]; setGoogleDriveFiles(mockFiles); } finally { setIsLoadingFiles(false); } }; // Handle file selection const handleFileSelect = (fileId: string) => { setSelectedFiles(prev => { if (prev.includes(fileId)) { return prev.filter(id => id !== fileId); } else { return [...prev, fileId]; } }); }; const handleFolderClick = (folderId: string, folderName: string) => { const sessionToken = localStorage.getItem('google_drive_session_token'); if (sessionToken) { setCurrentFolderId(folderId); setFolderPath(prev => [...prev, {id: folderId, name: folderName}]); loadGoogleDriveFiles(sessionToken, folderId); } }; const navigateBack = (index: number) => { const sessionToken = localStorage.getItem('google_drive_session_token'); if (sessionToken) { const newPath = folderPath.slice(0, index + 1); const targetFolderId = newPath[newPath.length - 1]?.id; setCurrentFolderId(targetFolderId); setFolderPath(newPath); loadGoogleDriveFiles(sessionToken, targetFolderId); } }; const handleSelectAll = () => { if (selectedFiles.length === googleDriveFiles.length) { setSelectedFiles([]); } else { setSelectedFiles(googleDriveFiles.map(file => file.id)); } }; const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, multiple: true, onDragEnter: doNothing, onDragOver: doNothing, onDragLeave: doNothing, maxSize: 25000000, accept: { 'application/pdf': ['.pdf'], 'text/plain': ['.txt'], 'text/x-rst': ['.rst'], 'text/x-markdown': ['.md'], 'application/zip': ['.zip'], 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'], 'application/json': ['.json'], 'text/csv': ['.csv'], 'text/html': ['.html'], 'application/epub+zip': ['.epub'], 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': [ '.xlsx', ], 'application/vnd.openxmlformats-officedocument.presentationml.presentation': ['.pptx'], 'image/png': ['.png'], 'image/jpeg': ['.jpeg'], 'image/jpg': ['.jpg'], }, }); const isUploadDisabled = (): boolean => { if (activeTab === 'file') { return !docName?.trim() || files.length === 0; } if (activeTab === 'remote') { if (!remoteName?.trim()) { return true; } if (ingestor.type === 'google_drive') { return !isGoogleDriveConnected || selectedFiles.length === 0; } const formFields: FormField[] = IngestorFormSchemas[ingestor.type]; for (const field of formFields) { if (field.required) { // Validate only required fields const value = ingestor.config[field.name as keyof typeof ingestor.config]; if (typeof value === 'string' && !value.trim()) { return true; } if ( typeof value === 'number' && (value === null || value === undefined || value <= 0) ) { return true; } if (typeof value === 'boolean' && value === undefined) { return true; } } } return false; } return true; }; const handleIngestorChange = ( key: keyof IngestorConfig['config'], value: string | number | boolean, ) => { setIngestor((prevState) => ({ ...prevState, config: { ...prevState.config, [key]: value, }, })); }; const handleIngestorTypeChange = (type: IngestorType) => { //Updates the ingestor seleced in dropdown and resets the config to the default config for that type const defaultConfig = IngestorDefaultConfigs[type]; setIngestor({ type, name: defaultConfig.name, config: defaultConfig.config, }); }; let view; if (progress?.type === 'UPLOAD') { view = ; } else if (progress?.type === 'TRAINING') { view = ; } else { view = (

{t('modals.uploadDoc.label')}

{!activeTab && (

{t('modals.uploadDoc.select')}

)} {activeTab === 'file' && ( <> setDocName(e.target.value)} borderVariant="thin" placeholder={t('modals.uploadDoc.name')} labelBgClassName="bg-white dark:bg-charleston-green-2" required={true} />
{t('modals.uploadDoc.choose')}

{t('modals.uploadDoc.info')}

{t('modals.uploadDoc.uploadedFiles')}

{files.map((file) => (

{file.name}

))} {files.length === 0 && (

{t('none')}

)}
)} {activeTab === 'remote' && ( <> opt.value === ingestor.type) || null } onSelect={(selected: { label: string; value: string }) => handleIngestorTypeChange(selected.value as IngestorType) } size="w-full" rounded="3xl" border="border" placeholder="Select ingestor type" placeholderClassName="text-gray-400 dark:text-silver" /> {/* Dynamically render form fields based on schema */} setRemoteName(e.target.value)} borderVariant="thin" placeholder="Name" required={true} labelBgClassName="bg-white dark:bg-charleston-green-2" /> {ingestor.type === 'google_drive' && (
{authError && (

⚠️ {authError}

)} {!isGoogleDriveConnected ? ( ) : (
{/* Connection Status */}
Connected as {userEmail}
{/* File Browser */}
{/* Breadcrumb navigation */}
{folderPath.map((path, index) => (
{index > 0 && /}
))}

Select Files from Google Drive

{googleDriveFiles.length > 0 && ( )}
{selectedFiles.length > 0 && (

{selectedFiles.length} file{selectedFiles.length !== 1 ? 's' : ''} selected

)}
{isLoadingFiles ? (
Loading files...
) : googleDriveFiles.length === 0 ? (
No files found in your Google Drive
) : (
{googleDriveFiles.map((file) => (
handleFileSelect(file.id)} className="h-4 w-4 text-blue-600 rounded border-gray-300 focus:ring-blue-500" />
{file.type === 'application/vnd.google-apps.folder' || file.isFolder ? (
handleFolderClick(file.id, file.name)} > Folder
) : (
File
)}

{ if (file.type === 'application/vnd.google-apps.folder' || file.isFolder) { handleFolderClick(file.id, file.name); } }} > {file.name}

{file.size} • Modified {file.modifiedTime}

))}
)}
)}
)} {renderFormFields()} {IngestorFormSchemas[ingestor.type].some( (field) => field.advanced, ) && ( )} )}
{activeTab && ( )} {activeTab && ( )}
); } return ( { close(); setDocName(''); setfiles([]); setModalState('INACTIVE'); setActiveTab(null); }} > {view} ); } export default Upload;