From 7896526f191517bf5c355cf201588c222510ea9c Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Fri, 5 Sep 2025 10:35:23 +0530 Subject: [PATCH 01/29] (feat:load_files) search feature --- application/api/connector/routes.py | 33 +++- .../parser/connectors/google_drive/loader.py | 29 ++- frontend/src/upload/Upload.tsx | 172 +++++++++++------- 3 files changed, 159 insertions(+), 75 deletions(-) diff --git a/application/api/connector/routes.py b/application/api/connector/routes.py index f203a703..5b12d8c4 100644 --- a/application/api/connector/routes.py +++ b/application/api/connector/routes.py @@ -339,8 +339,15 @@ class ConnectorRefresh(Resource): @connectors_ns.route("/api/connectors/files") class ConnectorFiles(Resource): - @api.expect(api.model("ConnectorFilesModel", {"provider": fields.String(required=True), "session_token": fields.String(required=True), "folder_id": fields.String(required=False), "limit": fields.Integer(required=False), "page_token": fields.String(required=False)})) - @api.doc(description="List files from a connector provider (supports pagination)") + @api.expect(api.model("ConnectorFilesModel", { + "provider": fields.String(required=True), + "session_token": fields.String(required=True), + "folder_id": fields.String(required=False), + "limit": fields.Integer(required=False), + "page_token": fields.String(required=False), + "search_query": fields.String(required=False) + })) + @api.doc(description="List files from a connector provider (supports pagination and search)") def post(self): try: data = request.get_json() @@ -349,10 +356,11 @@ class ConnectorFiles(Resource): folder_id = data.get('folder_id') limit = data.get('limit', 10) page_token = data.get('page_token') + search_query = data.get('search_query') + if not provider or not session_token: return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400) - decoded_token = request.decoded_token if not decoded_token: return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401) @@ -362,13 +370,17 @@ class ConnectorFiles(Resource): return make_response(jsonify({"success": False, "error": "Invalid or unauthorized session"}), 401) loader = ConnectorCreator.create_connector(provider, session_token) - documents = loader.load_data({ + input_config = { 'limit': limit, 'list_only': True, 'session_token': session_token, 'folder_id': folder_id, 'page_token': page_token - }) + } + if search_query: + input_config['search_query'] = search_query + + documents = loader.load_data(input_config) files = [] for doc in documents[:limit]: @@ -386,13 +398,20 @@ class ConnectorFiles(Resource): 'name': metadata.get('file_name', 'Unknown File'), 'type': metadata.get('mime_type', 'unknown'), 'size': metadata.get('size', None), - 'modifiedTime': formatted_time + 'modifiedTime': formatted_time, + 'isFolder': metadata.get('is_folder', False) }) next_token = getattr(loader, 'next_page_token', None) has_more = bool(next_token) - return make_response(jsonify({"success": True, "files": files, "total": len(files), "next_page_token": next_token, "has_more": has_more}), 200) + return make_response(jsonify({ + "success": True, + "files": files, + "total": len(files), + "next_page_token": next_token, + "has_more": has_more + }), 200) except Exception as e: current_app.logger.error(f"Error loading connector files: {e}") return make_response(jsonify({"success": False, "error": f"Failed to load files: {str(e)}"}), 500) diff --git a/application/parser/connectors/google_drive/loader.py b/application/parser/connectors/google_drive/loader.py index 07219344..7705a3c1 100644 --- a/application/parser/connectors/google_drive/loader.py +++ b/application/parser/connectors/google_drive/loader.py @@ -120,6 +120,7 @@ class GoogleDriveLoader(BaseConnectorLoader): list_only = inputs.get('list_only', False) load_content = not list_only page_token = inputs.get('page_token') + search_query = inputs.get('search_query') self.next_page_token = None if file_ids: @@ -128,12 +129,18 @@ class GoogleDriveLoader(BaseConnectorLoader): try: doc = self._load_file_by_id(file_id, load_content=load_content) if doc: - documents.append(doc) + if not search_query or ( + search_query.lower() in doc.extra_info.get('file_name', '').lower() + ): + documents.append(doc) elif hasattr(self, '_credential_refreshed') and self._credential_refreshed: self._credential_refreshed = False logging.info(f"Retrying load of file {file_id} after credential refresh") doc = self._load_file_by_id(file_id, load_content=load_content) - if doc: + if doc and ( + not search_query or + search_query.lower() in doc.extra_info.get('file_name', '').lower() + ): documents.append(doc) except Exception as e: logging.error(f"Error loading file {file_id}: {e}") @@ -141,7 +148,13 @@ class GoogleDriveLoader(BaseConnectorLoader): else: # Browsing mode: list immediate children of provided folder or root parent_id = folder_id if folder_id else 'root' - documents = self._list_items_in_parent(parent_id, limit=limit, load_content=load_content, page_token=page_token) + documents = self._list_items_in_parent( + parent_id, + limit=limit, + load_content=load_content, + page_token=page_token, + search_query=search_query + ) logging.info(f"Loaded {len(documents)} documents from Google Drive") return documents @@ -184,13 +197,18 @@ class GoogleDriveLoader(BaseConnectorLoader): return None - def _list_items_in_parent(self, parent_id: str, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None) -> List[Document]: + def _list_items_in_parent(self, parent_id: str, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]: self._ensure_service() documents: List[Document] = [] try: query = f"'{parent_id}' in parents and trashed=false" + + if search_query: + safe_search = search_query.replace("'", "\\'") + query += f" and name contains '{safe_search}'" + next_token_out: Optional[str] = None while True: @@ -205,7 +223,8 @@ class GoogleDriveLoader(BaseConnectorLoader): q=query, fields='nextPageToken,files(id,name,mimeType,size,createdTime,modifiedTime,parents)', pageToken=page_token, - pageSize=page_size + pageSize=page_size, + orderBy='name' ).execute() items = results.get('files', []) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 46a36f4c..3f0b5bfe 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -64,6 +64,7 @@ function Upload({ const [authError, setAuthError] = useState(''); const [currentFolderId, setCurrentFolderId] = useState(null); const [folderPath, setFolderPath] = useState>([{id: null, name: 'My Drive'}]); + const [searchQuery, setSearchQuery] = useState(''); const [nextPageToken, setNextPageToken] = useState(null); const [hasMoreFiles, setHasMoreFiles] = useState(false); @@ -549,7 +550,7 @@ function Upload({ setNextPageToken(null); setHasMoreFiles(false); - loadGoogleDriveFiles(sessionToken, null, null, false); + loadGoogleDriveFiles(sessionToken, null, undefined, ''); } else { removeSessionToken(ingestor.type); setIsGoogleDriveConnected(false); @@ -562,57 +563,63 @@ function Upload({ } }; - const loadGoogleDriveFiles = async ( - sessionToken: string, - folderId?: string | null, - pageToken?: string | null, - append: boolean = false, - ) => { - setIsLoadingFiles(true); + const loadGoogleDriveFiles = useCallback( + ( + sessionToken: string, + folderId: string | null, + pageToken?: string, + searchQuery: string = '' + ) => { - try { + setIsLoadingFiles(true); + const apiHost = import.meta.env.VITE_API_HOST; - const requestBody: any = { - session_token: sessionToken, - limit: 10, - }; - if (folderId) { - requestBody.folder_id = folderId; - } - if (pageToken) { - requestBody.page_token = pageToken; + if (!pageToken) { + setGoogleDriveFiles([]); } - const filesResponse = await fetch(`${apiHost}/api/connectors/files`, { + fetch(`${apiHost}/api/connectors/files`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${token}` }, - body: JSON.stringify({ ...requestBody, provider: 'google_drive' }) - }); - - if (!filesResponse.ok) { - throw new Error(`Failed to load files: ${filesResponse.status}`); - } - - const filesData = await filesResponse.json(); - - if (filesData.success && Array.isArray(filesData.files)) { - setGoogleDriveFiles(prev => append ? [...prev, ...filesData.files] : filesData.files); - setNextPageToken(filesData.next_page_token || null); - setHasMoreFiles(Boolean(filesData.has_more)); - } else { - throw new Error(filesData.error || 'Failed to load files'); - } - - } catch (error) { - console.error('Error loading Google Drive files:', error); - setAuthError(error instanceof Error ? error.message : 'Failed to load files. Please make sure your Google Drive account is properly connected and you granted offline access during authorization.'); - } finally { - setIsLoadingFiles(false); - } - }; + body: JSON.stringify({ + provider: 'google_drive', + session_token: sessionToken, + folder_id: folderId, + limit: 10, + page_token: pageToken, + search_query: searchQuery + }) + }) + .then(response => response.json()) + .then(data => { + if (data.success) { + setGoogleDriveFiles(prev => + pageToken ? [...prev, ...data.files] : data.files + ); + setNextPageToken(data.next_page_token); + setHasMoreFiles(!!data.next_page_token); + } else { + console.error('Error loading files:', data.error); + if (!pageToken) { + setGoogleDriveFiles([]); + } + } + }) + .catch(err => { + console.error('Error loading files:', err); + if (!pageToken) { + setGoogleDriveFiles([]); + } + }) + .finally(() => { + setIsLoadingFiles(false); + }); + }, + [token] + ); @@ -628,33 +635,38 @@ function Upload({ }; const handleFolderClick = (folderId: string, folderName: string) => { + if (folderId === currentFolderId) { + return; + } + + setIsLoadingFiles(true); + + setCurrentFolderId(folderId); + setFolderPath(prev => [...prev, { id: folderId, name: folderName }]); + + setSearchQuery(''); + const sessionToken = getSessionToken(ingestor.type); if (sessionToken) { - setCurrentFolderId(folderId); - setFolderPath(prev => [...prev, {id: folderId, name: folderName}]); - - setGoogleDriveFiles([]); - setNextPageToken(null); - setHasMoreFiles(false); - setSelectedFiles([]); - loadGoogleDriveFiles(sessionToken, folderId, null, false); + loadGoogleDriveFiles(sessionToken, folderId, undefined, ''); } }; const navigateBack = (index: number) => { - const sessionToken = getSessionToken(ingestor.type); - if (sessionToken) { + if (index < folderPath.length - 1) { const newPath = folderPath.slice(0, index + 1); - const targetFolderId = newPath[newPath.length - 1]?.id; - - setCurrentFolderId(targetFolderId as string | null); + const targetFolderId = newPath[newPath.length - 1].id; + + setIsLoadingFiles(true); + setFolderPath(newPath); - - setGoogleDriveFiles([]); - setNextPageToken(null); - setHasMoreFiles(false); - setSelectedFiles([]); - loadGoogleDriveFiles(sessionToken, targetFolderId ?? null, null, false); + setCurrentFolderId(targetFolderId); + + setSearchQuery(''); + const sessionToken = getSessionToken(ingestor.type); + if (sessionToken) { + loadGoogleDriveFiles(sessionToken, targetFolderId, undefined, ''); + } } }; @@ -957,6 +969,40 @@ function Upload({ ))} + {/* Search input */} +
+
+ setSearchQuery(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + const sessionToken = getSessionToken(ingestor.type); + if (sessionToken) { + loadGoogleDriveFiles(sessionToken, currentFolderId, undefined, searchQuery); + } + } + }} + className="w-full px-3 py-2 pr-10 text-sm border border-gray-300 rounded-lg focus:outline-none focus:ring-1 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:border-gray-600 dark:text-white" + /> + +
+
+

Select Files from Google Drive @@ -1131,7 +1177,7 @@ function Upload({ if (isNearBottom && hasMoreFiles && !isLoadingFiles && nextPageToken) { const sessionToken = getSessionToken(ingestor.type); if (sessionToken) { - loadGoogleDriveFiles(sessionToken, currentFolderId, nextPageToken, true); + loadGoogleDriveFiles(sessionToken, currentFolderId, nextPageToken); } } }; From 80406d0753afe9c120703f091fde509729b6cdc6 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Fri, 5 Sep 2025 13:25:36 +0530 Subject: [PATCH 02/29] (feat:drive) debounce search, ui/ux --- frontend/src/components/ToggleSwitch.tsx | 2 +- frontend/src/modals/WrapperModal.tsx | 6 +- frontend/src/upload/Upload.tsx | 180 +++++++++++------------ 3 files changed, 92 insertions(+), 96 deletions(-) diff --git a/frontend/src/components/ToggleSwitch.tsx b/frontend/src/components/ToggleSwitch.tsx index 69530e79..b27f4841 100644 --- a/frontend/src/components/ToggleSwitch.tsx +++ b/frontend/src/components/ToggleSwitch.tsx @@ -53,7 +53,7 @@ const ToggleSwitch: React.FC = ({ {label && ( {label} diff --git a/frontend/src/modals/WrapperModal.tsx b/frontend/src/modals/WrapperModal.tsx index 5c580d1c..a2826b61 100644 --- a/frontend/src/modals/WrapperModal.tsx +++ b/frontend/src/modals/WrapperModal.tsx @@ -42,10 +42,10 @@ export default function WrapperModal({ }, [close, isPerformingTask]); const modalContent = ( -
+
{!isPerformingTask && ( )} -
{children}
+
{children}
); diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 3f0b5bfe..44369162 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -30,6 +30,8 @@ import { } from './types/ingestor'; import FileIcon from '../assets/file.svg'; import FolderIcon from '../assets/folder.svg'; +import SearchIcon from '../assets/search.svg'; +import CheckIcon from '../assets/checkmark.svg'; import ConnectorAuth from '../components/ConnectorAuth'; function Upload({ @@ -69,6 +71,7 @@ function Upload({ const [nextPageToken, setNextPageToken] = useState(null); const [hasMoreFiles, setHasMoreFiles] = useState(false); const scrollContainerRef = useRef(null); + const searchTimeoutRef = useRef(null); const renderFormFields = () => { const schema = IngestorFormSchemas[ingestor.type]; @@ -192,7 +195,8 @@ function Upload({ checked, ); }} - className="mt-2" + size="small" + className={`mt-2 text-base`} /> ); default: @@ -621,7 +625,18 @@ function Upload({ [token] ); + const debouncedSearch = useCallback((query: string) => { + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + searchTimeoutRef.current = window.setTimeout(() => { + const sessionToken = getSessionToken(ingestor.type); + if (sessionToken) { + loadGoogleDriveFiles(sessionToken, currentFolderId, undefined, query); + } + }, 300); + }, [ingestor.type, currentFolderId, loadGoogleDriveFiles]); // Handle file selection const handleFileSelect = (fileId: string) => { @@ -670,14 +685,6 @@ function Upload({ } }; - const handleSelectAll = () => { - if (selectedFiles.length === googleDriveFiles.length) { - setSelectedFiles([]); - } else { - setSelectedFiles(googleDriveFiles.map(file => file.id)); - } - }; - const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, multiple: true, @@ -888,7 +895,7 @@ function Upload({
{authError && (
-

+

⚠️ {authError}

@@ -917,7 +924,7 @@ function Upload({ ) : (
{/* Connection Status */} -
+
@@ -944,7 +951,7 @@ function Upload({ body: JSON.stringify({ provider: ingestor.type, session_token: getSessionToken(ingestor.type) }) }).catch(err => console.error('Error disconnecting from Google Drive:', err)); }} - className="text-white hover:text-gray-200 text-xs underline" + className="text-[#212121] hover:text-gray-700 text-xs underline" > Disconnect @@ -952,7 +959,7 @@ function Upload({ {/* File Browser */}
-
+
{/* Breadcrumb navigation */}
{folderPath.map((path, index) => ( @@ -960,7 +967,7 @@ function Upload({ {index > 0 && /}
@@ -1007,23 +1002,16 @@ function Upload({

Select Files from Google Drive

- {googleDriveFiles.length > 0 && ( - - )} + + {selectedFiles.length > 0 + ? `${selectedFiles.length} file${selectedFiles.length !== 1 ? 's' : ''} selected` + : '' + } +
- {selectedFiles.length > 0 && ( -

- {selectedFiles.length} file{selectedFiles.length !== 1 ? 's' : ''} selected -

- )}
-
+
{isLoadingFiles && googleDriveFiles.length === 0 ? (
@@ -1041,66 +1029,69 @@ function Upload({ {googleDriveFiles.map((file) => (
-
-
- handleFileSelect(file.id)} - className="h-4 w-4 text-blue-600 rounded border-gray-300 focus:ring-blue-500" - /> -
- {file.type === 'application/vnd.google-apps.folder' || file.isFolder ? ( +
+
{ + e.stopPropagation(); + handleFileSelect(file.id); + }} + >
handleFolderClick(file.id, file.name)} + className="flex h-5 w-5 shrink-0 items-center justify-center border border-[#C6C6C6] p-[0.5px] dark:border-[#757783] cursor-pointer" > - Folder + {selectedFiles.includes(file.id) && ( + Selected + )}
- ) : ( -
- File +
+
{ + if (file.type === 'application/vnd.google-apps.folder' || file.isFolder) { + handleFolderClick(file.id, file.name); + } else { + handleFileSelect(file.id); + } + }} + > +
+ {file.type +
+
+

+ {file.name} +

+

+ {file.size && `${formatBytes(file.size)} • `}Modified {formatDate(file.modifiedTime)} +

- )} -
-

{ - if (file.type === 'application/vnd.google-apps.folder' || file.isFolder) { - handleFolderClick(file.id, file.name); - } - }} - > - {file.name} -

-

- {file.size && `${formatBytes(file.size)} • `}Modified {formatDate(file.modifiedTime)} -

))}
-
- {isLoadingFiles && ( + {isLoadingFiles && ( +
Loading more files...
- )} - {!hasMoreFiles && !isLoadingFiles && ( - All files loaded - )} -
+
+ )} )}
@@ -1154,10 +1145,7 @@ function Upload({ : 'bg-purple-30 hover:bg-violets-are-blue cursor-pointer text-white' }`} > - {ingestor.type === 'google_drive' && selectedFiles.length > 0 - ? `Train with ${selectedFiles.length} file${selectedFiles.length !== 1 ? 's' : ''}` - : t('modals.uploadDoc.train') - } + {t('modals.uploadDoc.train')} )}
@@ -1189,6 +1177,14 @@ function Upload({ }; }, [hasMoreFiles, isLoadingFiles, nextPageToken, currentFolderId, ingestor.type]); + useEffect(() => { + return () => { + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + }; + }, []); + return ( Date: Mon, 8 Sep 2025 19:09:19 +0530 Subject: [PATCH 03/29] (refactor:uploads) separate file picker --- frontend/src/components/FilePicker.tsx | 480 ++++++++++++++++++++++++ frontend/src/upload/Upload.tsx | 498 +++---------------------- 2 files changed, 524 insertions(+), 454 deletions(-) create mode 100644 frontend/src/components/FilePicker.tsx diff --git a/frontend/src/components/FilePicker.tsx b/frontend/src/components/FilePicker.tsx new file mode 100644 index 00000000..8c95c6a2 --- /dev/null +++ b/frontend/src/components/FilePicker.tsx @@ -0,0 +1,480 @@ +import React, { useState, useEffect, useCallback, useRef } from 'react'; +import { formatBytes } from '../utils/stringUtils'; +import { formatDate } from '../utils/dateTimeUtils'; +import { getSessionToken, setSessionToken, removeSessionToken } from '../utils/providerUtils'; +import ConnectorAuth from '../components/ConnectorAuth'; +import FileIcon from '../assets/file.svg'; +import FolderIcon from '../assets/folder.svg'; +import CheckIcon from '../assets/checkmark.svg'; +import SearchIcon from '../assets/search.svg'; + +interface CloudFile { + id: string; + name: string; + type: string; + size?: number; + modifiedTime: string; + isFolder?: boolean; +} + +interface CloudFilePickerProps { + onSelectionChange: (selectedFileIds: string[], selectedFolderIds?: string[]) => void; + onDisconnect?: () => void; + provider: string; + token: string | null; + initialSelectedFiles?: string[]; + initialSelectedFolders?: string[]; +} + +export const FilePicker: React.FC = ({ + onSelectionChange, + onDisconnect, + provider, + token, + initialSelectedFiles = [], +}) => { + const [files, setFiles] = useState([]); + const [selectedFiles, setSelectedFiles] = useState(initialSelectedFiles); + const [selectedFolders, setSelectedFolders] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [hasMoreFiles, setHasMoreFiles] = useState(false); + const [nextPageToken, setNextPageToken] = useState(null); + const [currentFolderId, setCurrentFolderId] = useState(null); + const [folderPath, setFolderPath] = useState>([{ + id: null, + name: 'Drive' + }]); + const [searchQuery, setSearchQuery] = useState(''); + const [authError, setAuthError] = useState(''); + const [isConnected, setIsConnected] = useState(false); + + const scrollContainerRef = useRef(null); + const searchTimeoutRef = useRef | null>(null); + + const isFolder = (file: CloudFile) => { + return file.isFolder || + file.type === 'application/vnd.google-apps.folder' || + file.type === 'folder'; + }; + + const providerDisplayNames = { + google_drive: 'Drive', + }; + + const getConnectorDisplayName = (provider: string) => { + return providerDisplayNames[provider as keyof typeof providerDisplayNames] || provider; + }; + + const loadCloudFiles = useCallback( + async ( + sessionToken: string, + folderId: string | null, + pageToken?: string, + searchQuery: string = '' + ) => { + setIsLoading(true); + + const apiHost = import.meta.env.VITE_API_HOST; + if (!pageToken) { + setFiles([]); + } + + try { + const response = await fetch(`${apiHost}/api/connectors/files`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}` + }, + body: JSON.stringify({ + provider: provider, + session_token: sessionToken, + folder_id: folderId, + limit: 10, + page_token: pageToken, + search_query: searchQuery + }) + }); + + const data = await response.json(); + if (data.success) { + setFiles(prev => pageToken ? [...prev, ...data.files] : data.files); + setNextPageToken(data.next_page_token); + setHasMoreFiles(!!data.next_page_token); + } else { + console.error('Error loading files:', data.error); + if (!pageToken) { + setFiles([]); + } + } + } catch (err) { + console.error('Error loading files:', err); + if (!pageToken) { + setFiles([]); + } + } finally { + setIsLoading(false); + } + }, + [token, provider] + ); + + const validateAndLoadFiles = useCallback(async () => { + const sessionToken = getSessionToken(provider); + if (!sessionToken) { + setIsConnected(false); + return; + } + + try { + const apiHost = import.meta.env.VITE_API_HOST; + const validateResponse = await fetch(`${apiHost}/api/connectors/validate-session`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}` + }, + body: JSON.stringify({ provider: provider, session_token: sessionToken }) + }); + + if (!validateResponse.ok) { + removeSessionToken(provider); + setIsConnected(false); + setAuthError('Session expired. Please reconnect to Google Drive.'); + return; + } + + const validateData = await validateResponse.json(); + if (validateData.success) { + + setIsConnected(true); + setAuthError(''); + + setFiles([]); + setNextPageToken(null); + setHasMoreFiles(false); + setCurrentFolderId(null); + setFolderPath([{id: null, name: provider === 'google_drive' ? 'My Drive' : + provider === 'onedrive' ? 'My OneDrive' : + provider === 'sharepoint' ? 'SharePoint' : 'Root'}]); + loadCloudFiles(sessionToken, null, undefined, ''); + } else { + removeSessionToken(provider); + setIsConnected(false); + setAuthError(validateData.error || 'Session expired. Please reconnect your account.'); + } + } catch (error) { + console.error('Error validating session:', error); + setAuthError('Failed to validate session. Please reconnect.'); + setIsConnected(false); + } + }, [provider, token, loadCloudFiles]); + + useEffect(() => { + validateAndLoadFiles(); + }, [validateAndLoadFiles]); + + const handleScroll = useCallback(() => { + const scrollContainer = scrollContainerRef.current; + if (!scrollContainer) return; + + const { scrollTop, scrollHeight, clientHeight } = scrollContainer; + const isNearBottom = scrollHeight - scrollTop - clientHeight < 50; + + if (isNearBottom && hasMoreFiles && !isLoading && nextPageToken) { + const sessionToken = getSessionToken(provider); + if (sessionToken) { + loadCloudFiles(sessionToken, currentFolderId, nextPageToken, searchQuery); + } + } + }, [hasMoreFiles, isLoading, nextPageToken, currentFolderId, searchQuery, provider, loadCloudFiles]); + + useEffect(() => { + const scrollContainer = scrollContainerRef.current; + if (scrollContainer) { + scrollContainer.addEventListener('scroll', handleScroll); + return () => scrollContainer.removeEventListener('scroll', handleScroll); + } + }, [handleScroll]); + + useEffect(() => { + return () => { + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + }; + }, []); + + const handleSearchChange = (query: string) => { + setSearchQuery(query); + + if (searchTimeoutRef.current) { + clearTimeout(searchTimeoutRef.current); + } + + searchTimeoutRef.current = setTimeout(() => { + const sessionToken = getSessionToken(provider); + if (sessionToken) { + loadCloudFiles(sessionToken, currentFolderId, undefined, query); + } + }, 300); + }; + + const handleFolderClick = (folderId: string, folderName: string) => { + if (folderId === currentFolderId) { + return; + } + + setIsLoading(true); + + setCurrentFolderId(folderId); + setFolderPath(prev => [...prev, { id: folderId, name: folderName }]); + setSearchQuery(''); + + const sessionToken = getSessionToken(provider); + if (sessionToken) { + loadCloudFiles(sessionToken, folderId, undefined, ''); + } + }; + + const navigateBack = (index: number) => { + if (index >= folderPath.length - 1) return; + + const newFolderPath = folderPath.slice(0, index + 1); + const newFolderId = newFolderPath[newFolderPath.length - 1].id; + + setFolderPath(newFolderPath); + setCurrentFolderId(newFolderId); + setSearchQuery(''); + + const sessionToken = getSessionToken(provider); + if (sessionToken) { + loadCloudFiles(sessionToken, newFolderId, undefined, ''); + } + }; + + const handleFileSelect = (fileId: string, isFolder: boolean) => { + if (isFolder) { + const newSelectedFolders = selectedFolders.includes(fileId) + ? selectedFolders.filter(id => id !== fileId) + : [...selectedFolders, fileId]; + setSelectedFolders(newSelectedFolders); + onSelectionChange(selectedFiles, newSelectedFolders); + } else { + const newSelectedFiles = selectedFiles.includes(fileId) + ? selectedFiles.filter(id => id !== fileId) + : [...selectedFiles, fileId]; + setSelectedFiles(newSelectedFiles); + onSelectionChange(newSelectedFiles, selectedFolders); + } + }; + + // Render authentication UI + if (!isConnected) { + return ( +
+ {authError && ( +
{authError}
+ )} + { + setIsConnected(true); + setAuthError(''); + + if (data.session_token) { + setSessionToken(provider, data.session_token); + loadCloudFiles(data.session_token, null); + } + }} + onError={(error) => { + setAuthError(error); + setIsConnected(false); + }} + /> +
+ ); + } + + // Render file browser UI + return ( +
+ {/* Connected state indicator */} +
+
+
+ + + + Connected to {getConnectorDisplayName(provider)} +
+ +
+
+ +
+
+ {/* Breadcrumb navigation */} +
+ {folderPath.map((path, index) => ( +
+ {index > 0 && /} + +
+ ))} +
+ + {/* Search input */} +
+
+ handleSearchChange(e.target.value)} + className="w-full px-3 py-2 pr-10 text-sm border border-gray-300 rounded-lg focus:outline-none focus:ring-1 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:border-gray-600 dark:text-white" + /> + +
+
+ +
+

+ Select Files from {getConnectorDisplayName(provider)} +

+ + {selectedFiles.length + selectedFolders.length > 0 + ? `${selectedFiles.length + selectedFolders.length} item${(selectedFiles.length + selectedFolders.length) !== 1 ? 's' : ''} selected` + : '' + } + +
+
+ +
+ {isLoading && files.length === 0 ? ( +
+
+
+ Loading files... +
+
+ ) : files.length === 0 ? ( +
+ No files found in your {getConnectorDisplayName(provider)} +
+ ) : ( + <> +
+ {files.map((file, index) => ( +
+
+
{ + e.stopPropagation(); + handleFileSelect(file.id, isFolder(file)); + }} + > +
+ {(isFolder(file) ? selectedFolders : selectedFiles).includes(file.id) && ( + Selected + )} +
+
+
{ + if (isFolder(file)) { + handleFolderClick(file.id, file.name); + } else { + handleFileSelect(file.id, false); + } + }} + > +
+ {isFolder(file) +
+
+

+ {file.name} +

+

+ {file.size && `${formatBytes(file.size)} • `}Modified {formatDate(file.modifiedTime)} +

+
+
+
+
+ ))} +
+ + {isLoading && ( +
+
+
+ Loading more files... +
+
+ )} + + )} +
+
+
+ ); +}; diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 44369162..76a6cd36 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -1,12 +1,10 @@ -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { useDropzone } from 'react-dropzone'; import { useTranslation } from 'react-i18next'; import { useDispatch, useSelector } from 'react-redux'; import userService from '../api/services/userService'; -import { getSessionToken, setSessionToken, removeSessionToken } from '../utils/providerUtils'; -import { formatDate } from '../utils/dateTimeUtils'; -import { formatBytes } from '../utils/stringUtils'; +import { getSessionToken } from '../utils/providerUtils'; import FileUpload from '../assets/file_upload.svg'; import WebsiteCollect from '../assets/website_collect.svg'; import Dropdown from '../components/Dropdown'; @@ -28,11 +26,8 @@ import { IngestorFormSchemas, IngestorType, } from './types/ingestor'; -import FileIcon from '../assets/file.svg'; -import FolderIcon from '../assets/folder.svg'; -import SearchIcon from '../assets/search.svg'; -import CheckIcon from '../assets/checkmark.svg'; -import ConnectorAuth from '../components/ConnectorAuth'; + +import {FilePicker} from '../components/FilePicker'; function Upload({ receivedFile = [], @@ -56,22 +51,17 @@ function Upload({ const [activeTab, setActiveTab] = useState(renderTab); const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); - // Google Drive state - const [isGoogleDriveConnected, setIsGoogleDriveConnected] = useState(false); - const [googleDriveFiles, setGoogleDriveFiles] = useState([]); + // Connector state const [selectedFiles, setSelectedFiles] = useState([]); - const [isLoadingFiles, setIsLoadingFiles] = useState(false); - const [isAuthenticating, setIsAuthenticating] = useState(false); - const [userEmail, setUserEmail] = useState(''); - const [authError, setAuthError] = useState(''); - const [currentFolderId, setCurrentFolderId] = useState(null); - const [folderPath, setFolderPath] = useState>([{id: null, name: 'My Drive'}]); - const [searchQuery, setSearchQuery] = useState(''); + const [selectedFolders, setSelectedFolders] = useState([]); - const [nextPageToken, setNextPageToken] = useState(null); - const [hasMoreFiles, setHasMoreFiles] = useState(false); - const scrollContainerRef = useRef(null); - const searchTimeoutRef = useRef(null); + // Helper function to check if ingestor type is a connector + const isConnectorType = (type: string) => { + return type === 'google_drive' || type === 'onedrive' || type === 'sharepoint'; + }; + + + const renderFormFields = () => { const schema = IngestorFormSchemas[ingestor.type]; @@ -456,26 +446,16 @@ function Upload({ let configData; - if (ingestor.type === 'google_drive') { + if (isConnectorType(ingestor.type)) { const sessionToken = getSessionToken(ingestor.type); - const selectedItems = googleDriveFiles.filter(file => selectedFiles.includes(file.id)); - const selectedFolderIds = selectedItems - .filter(item => item.type === 'application/vnd.google-apps.folder' || item.isFolder) - .map(folder => folder.id); - - const selectedFileIds = selectedItems - .filter(item => item.type !== 'application/vnd.google-apps.folder' && !item.isFolder) - .map(file => file.id); - configData = { - file_ids: selectedFileIds, - folder_ids: selectedFolderIds, - recursive: ingestor.config.recursive, - session_token: sessionToken || null + provider: ingestor.type, + session_token: sessionToken, + file_ids: selectedFiles, + folder_ids: selectedFolders, }; } else { - configData = { ...ingestor.config }; } @@ -507,185 +487,24 @@ function Upload({ xhr.send(formData); }; - useEffect(() => { - if (ingestor.type === 'google_drive') { - const sessionToken = getSessionToken(ingestor.type); - - if (sessionToken) { - // Auto-authenticate if session token exists - setIsGoogleDriveConnected(true); - setAuthError(''); - - // Fetch user email and files using the existing session token - - fetchUserEmailAndLoadFiles(sessionToken); - } - } - }, [ingestor.type]); - - const fetchUserEmailAndLoadFiles = async (sessionToken: string) => { - try { - const apiHost = import.meta.env.VITE_API_HOST; - - const validateResponse = await fetch(`${apiHost}/api/connectors/validate-session`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${token}` - }, - body: JSON.stringify({ provider: 'google_drive', session_token: sessionToken }) - }); - - if (!validateResponse.ok) { - removeSessionToken(ingestor.type); - setIsGoogleDriveConnected(false); - setAuthError('Session expired. Please reconnect to Google Drive.'); - return; - } - - const validateData = await validateResponse.json(); - - if (validateData.success) { - setUserEmail(validateData.user_email || 'Connected User'); - // reset pagination state and files - setGoogleDriveFiles([]); + - setNextPageToken(null); - setHasMoreFiles(false); - loadGoogleDriveFiles(sessionToken, null, undefined, ''); - } else { - removeSessionToken(ingestor.type); - setIsGoogleDriveConnected(false); - setAuthError(validateData.error || 'Session expired. Please reconnect your Google Drive account and make sure to grant offline access.'); - } - } catch (error) { - console.error('Error validating Google Drive session:', error); - setAuthError('Failed to validate session. Please reconnect.'); - setIsGoogleDriveConnected(false); - } - }; - - const loadGoogleDriveFiles = useCallback( - ( - sessionToken: string, - folderId: string | null, - pageToken?: string, - searchQuery: string = '' - ) => { - - setIsLoadingFiles(true); - const apiHost = import.meta.env.VITE_API_HOST; - if (!pageToken) { - setGoogleDriveFiles([]); - } - - fetch(`${apiHost}/api/connectors/files`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${token}` - }, - body: JSON.stringify({ - provider: 'google_drive', - session_token: sessionToken, - folder_id: folderId, - limit: 10, - page_token: pageToken, - search_query: searchQuery - }) - }) - .then(response => response.json()) - .then(data => { - if (data.success) { - setGoogleDriveFiles(prev => - pageToken ? [...prev, ...data.files] : data.files - ); - setNextPageToken(data.next_page_token); - setHasMoreFiles(!!data.next_page_token); - } else { - console.error('Error loading files:', data.error); - if (!pageToken) { - setGoogleDriveFiles([]); - } - } - }) - .catch(err => { - console.error('Error loading files:', err); - if (!pageToken) { - setGoogleDriveFiles([]); - } - }) - .finally(() => { - setIsLoadingFiles(false); - }); - }, - [token] - ); - - const debouncedSearch = useCallback((query: string) => { - if (searchTimeoutRef.current) { - clearTimeout(searchTimeoutRef.current); - } - - searchTimeoutRef.current = window.setTimeout(() => { - const sessionToken = getSessionToken(ingestor.type); - if (sessionToken) { - loadGoogleDriveFiles(sessionToken, currentFolderId, undefined, query); - } - }, 300); - }, [ingestor.type, currentFolderId, loadGoogleDriveFiles]); - - // Handle file selection - const handleFileSelect = (fileId: string) => { - setSelectedFiles(prev => { - if (prev.includes(fileId)) { - return prev.filter(id => id !== fileId); - } else { - return [...prev, fileId]; - } - }); - }; - - const handleFolderClick = (folderId: string, folderName: string) => { - if (folderId === currentFolderId) { - return; - } - - setIsLoadingFiles(true); - - setCurrentFolderId(folderId); - setFolderPath(prev => [...prev, { id: folderId, name: folderName }]); - - setSearchQuery(''); - - const sessionToken = getSessionToken(ingestor.type); - if (sessionToken) { - loadGoogleDriveFiles(sessionToken, folderId, undefined, ''); - } - }; - - const navigateBack = (index: number) => { - if (index < folderPath.length - 1) { - const newPath = folderPath.slice(0, index + 1); - const targetFolderId = newPath[newPath.length - 1].id; - setIsLoadingFiles(true); - setFolderPath(newPath); - setCurrentFolderId(targetFolderId); - - setSearchQuery(''); - const sessionToken = getSessionToken(ingestor.type); - if (sessionToken) { - loadGoogleDriveFiles(sessionToken, targetFolderId, undefined, ''); - } - } - }; + - const { getRootProps, getInputProps, isDragActive } = useDropzone({ + + + + + + + + + const { getRootProps, getInputProps } = useDropzone({ onDrop, multiple: true, onDragEnter: doNothing, @@ -723,8 +542,8 @@ function Upload({ if (!remoteName?.trim()) { return true; } - if (ingestor.type === 'google_drive') { - return !isGoogleDriveConnected || selectedFiles.length === 0; + if (isConnectorType(ingestor.type)) { + return selectedFiles.length === 0; } const formFields: FormField[] = IngestorFormSchemas[ingestor.type]; @@ -891,218 +710,17 @@ function Upload({ required={true} labelBgClassName="bg-white dark:bg-charleston-green-2" /> - {ingestor.type === 'google_drive' && ( -
- {authError && ( -
-

- ⚠️ {authError} -

-
- )} - - {!isGoogleDriveConnected ? ( - { - setUserEmail(data.user_email); - setIsGoogleDriveConnected(true); - setIsAuthenticating(false); - setAuthError(''); - - if (data.session_token) { - setSessionToken(ingestor.type, data.session_token); - loadGoogleDriveFiles(data.session_token, null); - } - }} - onError={(error) => { - setAuthError(error); - setIsAuthenticating(false); - setIsGoogleDriveConnected(false); - }} - /> - ) : ( -
- {/* Connection Status */} -
-
- - - - Connected as {userEmail} -
- -
- - {/* File Browser */} -
-
- {/* Breadcrumb navigation */} -
- {folderPath.map((path, index) => ( -
- {index > 0 && /} - -
- ))} -
- - {/* Search input */} -
-
- { - const newQuery = e.target.value; - setSearchQuery(newQuery); - debouncedSearch(newQuery); - }} - className="w-full px-3 py-2 pr-10 text-sm border border-gray-300 rounded-lg focus:outline-none focus:ring-1 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:border-gray-600 dark:text-white" - /> - -
-
- -
-

- Select Files from Google Drive -

- - {selectedFiles.length > 0 - ? `${selectedFiles.length} file${selectedFiles.length !== 1 ? 's' : ''} selected` - : '' - } - -
-
- -
- {isLoadingFiles && googleDriveFiles.length === 0 ? ( -
-
-
- Loading files... -
-
- ) : googleDriveFiles.length === 0 ? ( -
- No files found in your Google Drive -
- ) : ( - <> -
- {googleDriveFiles.map((file) => ( -
-
-
{ - e.stopPropagation(); - handleFileSelect(file.id); - }} - > -
- {selectedFiles.includes(file.id) && ( - Selected - )} -
-
-
{ - if (file.type === 'application/vnd.google-apps.folder' || file.isFolder) { - handleFolderClick(file.id, file.name); - } else { - handleFileSelect(file.id); - } - }} - > -
- {file.type -
-
-

- {file.name} -

-

- {file.size && `${formatBytes(file.size)} • `}Modified {formatDate(file.modifiedTime)} -

-
-
-
-
- ))} -
- - {isLoadingFiles && ( -
-
-
- Loading more files... -
-
- )} - - )} -
- - - -
-
- )} -
+ {isConnectorType(ingestor.type) && ( + { + setSelectedFiles(selectedFileIds); + setSelectedFolders(selectedFolderIds); + }} + provider={ingestor.type} + token={token} + initialSelectedFiles={selectedFiles} + initialSelectedFolders={selectedFolders} + /> )} {renderFormFields()} @@ -1153,37 +771,9 @@ function Upload({ ); } - useEffect(() => { - const scrollContainer = scrollContainerRef.current; - - const handleScroll = () => { - if (!scrollContainer) return; - - const { scrollTop, scrollHeight, clientHeight } = scrollContainer; - const isNearBottom = scrollHeight - scrollTop - clientHeight < 50; - - if (isNearBottom && hasMoreFiles && !isLoadingFiles && nextPageToken) { - const sessionToken = getSessionToken(ingestor.type); - if (sessionToken) { - loadGoogleDriveFiles(sessionToken, currentFolderId, nextPageToken); - } - } - }; - - scrollContainer?.addEventListener('scroll', handleScroll); - - return () => { - scrollContainer?.removeEventListener('scroll', handleScroll); - }; - }, [hasMoreFiles, isLoadingFiles, nextPageToken, currentFolderId, ingestor.type]); + - useEffect(() => { - return () => { - if (searchTimeoutRef.current) { - clearTimeout(searchTimeoutRef.current); - } - }; - }, []); + return ( Date: Tue, 9 Sep 2025 00:00:58 +0530 Subject: [PATCH 04/29] (feat:connectors) type as connector:file --- application/api/connector/routes.py | 2 +- application/worker.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/api/connector/routes.py b/application/api/connector/routes.py index 5b12d8c4..f1ba247b 100644 --- a/application/api/connector/routes.py +++ b/application/api/connector/routes.py @@ -173,7 +173,7 @@ class ConnectorSources(Resource): return make_response(jsonify({"success": False}), 401) user = decoded_token.get("sub") try: - sources = sources_collection.find({"user": user, "type": "connector"}).sort("date", -1) + sources = sources_collection.find({"user": user, "type": {"$regex": "^connector:"}}).sort("date", -1) connector_sources = [] for source in sources: connector_sources.append({ diff --git a/application/worker.py b/application/worker.py index 10fb6c2b..5a29d00a 100755 --- a/application/worker.py +++ b/application/worker.py @@ -978,13 +978,13 @@ def ingest_connector( "tokens": tokens, "retriever": retriever, "id": str(id), - "type": "connector", + "type": "connector:file", "remote_data": json.dumps({ "provider": source_type, **api_source_config }), "directory_structure": json.dumps(directory_structure), - "sync_frequency": sync_frequency + "sync_frequency": sync_frequency } if operation_mode == "sync": From e749c936c989a9014599d7a1b4943a28dbaa9f4a Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Tue, 9 Sep 2025 13:07:26 +0530 Subject: [PATCH 05/29] (refactor:uploads) for new ui --- frontend/src/upload/Upload.tsx | 137 +++++++++++++++++++++----- frontend/src/upload/types/ingestor.ts | 33 ++++++- 2 files changed, 141 insertions(+), 29 deletions(-) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 76a6cd36..3c42eca0 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -51,15 +51,10 @@ function Upload({ const [activeTab, setActiveTab] = useState(renderTab); const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); - // Connector state + // File picker state const [selectedFiles, setSelectedFiles] = useState([]); const [selectedFolders, setSelectedFolders] = useState([]); - // Helper function to check if ingestor type is a connector - const isConnectorType = (type: string) => { - return type === 'google_drive' || type === 'onedrive' || type === 'sharepoint'; - }; - @@ -189,6 +184,52 @@ function Upload({ className={`mt-2 text-base`} /> ); + case 'file_picker': + return ( + { + setSelectedFiles(selectedFileIds); + setSelectedFolders(selectedFolderIds); + }} + provider={ingestor.type} + token={token} + initialSelectedFiles={selectedFiles} + initialSelectedFolders={selectedFolders} + /> + ); + case 'local_file_picker': + return ( +
+
+ + + Choose Files + +
+
+

+ Selected Files +

+
+ {files.map((file) => ( +

+ {file.name} +

+ ))} + {files.length === 0 && ( +

+ No files selected +

+ )} +
+
+
+ ); default: return null; } @@ -221,6 +262,7 @@ function Upload({ { label: 'GitHub', value: 'github' }, { label: 'Reddit', value: 'reddit' }, { label: 'Google Drive', value: 'google_drive' }, + { label: 'Upload File', value: 'local_file' }, ]; const sourceDocs = useSelector(selectSourceDocs); @@ -409,7 +451,18 @@ function Upload({ const onDrop = useCallback((acceptedFiles: File[]) => { setfiles(acceptedFiles); setDocName(acceptedFiles[0]?.name || ''); - }, []); + + // If we're in local_file mode, update the ingestor config + if (ingestor.type === 'local_file') { + setIngestor((prevState) => ({ + ...prevState, + config: { + ...prevState.config, + files: acceptedFiles, + }, + })); + } + }, [ingestor.type]); const doNothing = () => undefined; @@ -446,7 +499,15 @@ function Upload({ let configData; - if (isConnectorType(ingestor.type)) { + const filePickerField = IngestorFormSchemas[ingestor.type].find( + (field) => field.type === 'file_picker' + ); + + const localFilePickerField = IngestorFormSchemas[ingestor.type].find( + (field) => field.type === 'local_file_picker' + ); + + if (filePickerField) { const sessionToken = getSessionToken(ingestor.type); configData = { @@ -455,6 +516,13 @@ function Upload({ file_ids: selectedFiles, folder_ids: selectedFolders, }; + } else if (localFilePickerField) { + // For local files, we need to handle them differently + // Instead of sending config data, we'll append the files directly to formData + files.forEach((file) => { + formData.append('file', file); + }); + configData = { ...ingestor.config }; } else { configData = { ...ingestor.config }; } @@ -482,7 +550,11 @@ function Upload({ }); }, 3000); }; - xhr.open('POST', `${apiHost}/api/remote`); + + // Use different endpoints based on ingestor type + const endpoint = ingestor.type === 'local_file' ? `${apiHost}/api/upload` : `${apiHost}/api/remote`; + + xhr.open('POST', endpoint); xhr.setRequestHeader('Authorization', `Bearer ${token}`); xhr.send(formData); }; @@ -542,8 +614,21 @@ function Upload({ if (!remoteName?.trim()) { return true; } - if (isConnectorType(ingestor.type)) { - return selectedFiles.length === 0; + const filePickerField = IngestorFormSchemas[ingestor.type].find( + (field) => field.type === 'file_picker' + ); + + if (filePickerField?.required && selectedFiles.length === 0 && selectedFolders.length === 0) { + return true; + } + + // Check for local file picker + const localFilePickerField = IngestorFormSchemas[ingestor.type].find( + (field) => field.type === 'local_file_picker' + ); + + if (localFilePickerField?.required && files.length === 0) { + return true; } const formFields: FormField[] = IngestorFormSchemas[ingestor.type]; @@ -594,6 +679,14 @@ function Upload({ name: defaultConfig.name, config: defaultConfig.config, }); + + // Sync remoteName with ingestor name + setRemoteName(defaultConfig.name); + + // Clear files if switching away from local_file + if (type !== 'local_file') { + setfiles([]); + } }; let view; @@ -704,25 +797,19 @@ function Upload({ type="text" colorVariant="silver" value={remoteName} - onChange={(e) => setRemoteName(e.target.value)} + onChange={(e) => { + setRemoteName(e.target.value); + // Also update the ingestor name + setIngestor((prevState) => ({ + ...prevState, + name: e.target.value, + })); + }} borderVariant="thin" placeholder="Name" required={true} labelBgClassName="bg-white dark:bg-charleston-green-2" /> - {isConnectorType(ingestor.type) && ( - { - setSelectedFiles(selectedFileIds); - setSelectedFolders(selectedFolderIds); - }} - provider={ingestor.type} - token={token} - initialSelectedFiles={selectedFiles} - initialSelectedFolders={selectedFolders} - /> - )} - {renderFormFields()} {IngestorFormSchemas[ingestor.type].some( (field) => field.advanced, diff --git a/frontend/src/upload/types/ingestor.ts b/frontend/src/upload/types/ingestor.ts index f5c29dee..ed129ad2 100644 --- a/frontend/src/upload/types/ingestor.ts +++ b/frontend/src/upload/types/ingestor.ts @@ -1,5 +1,5 @@ export interface BaseIngestorConfig { - [key: string]: string | number | boolean | undefined; + [key: string]: string | number | boolean | undefined | File[]; } export interface RedditIngestorConfig extends BaseIngestorConfig { @@ -29,7 +29,11 @@ export interface GoogleDriveIngestorConfig extends BaseIngestorConfig { token_info?: any; } -export type IngestorType = 'crawler' | 'github' | 'reddit' | 'url' | 'google_drive'; +export interface LocalFileIngestorConfig extends BaseIngestorConfig { + files: File[]; +} + +export type IngestorType = 'crawler' | 'github' | 'reddit' | 'url' | 'google_drive' | 'local_file'; export interface IngestorConfig { type: IngestorType; @@ -39,7 +43,8 @@ export interface IngestorConfig { | GithubIngestorConfig | CrawlerIngestorConfig | UrlIngestorConfig - | GoogleDriveIngestorConfig; + | GoogleDriveIngestorConfig + | LocalFileIngestorConfig; } export type IngestorFormData = { @@ -49,7 +54,7 @@ export type IngestorFormData = { data: string; }; -export type FieldType = 'string' | 'number' | 'enum' | 'boolean'; +export type FieldType = 'string' | 'number' | 'enum' | 'boolean' | 'file_picker' | 'local_file_picker'; export interface FormField { name: string; @@ -118,6 +123,12 @@ export const IngestorFormSchemas: Record = { }, ], google_drive: [ + { + name: 'file_picker', + label: 'Select files', + type: 'file_picker', + required: true, + }, { name: 'recursive', label: 'Include subfolders', @@ -125,6 +136,14 @@ export const IngestorFormSchemas: Record = { required: false, }, ], + local_file: [ + { + name: 'files', + label: 'Select files', + type: 'local_file_picker', + required: true, + }, + ], }; export const IngestorDefaultConfigs: Record< @@ -167,4 +186,10 @@ export const IngestorDefaultConfigs: Record< recursive: true, } as GoogleDriveIngestorConfig, }, + local_file: { + name: '', + config: { + files: [], + } as LocalFileIngestorConfig, + }, }; From fc8be45d5abc376c673900eb5f3c5123ea9c37d0 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Tue, 9 Sep 2025 13:08:38 +0530 Subject: [PATCH 06/29] (feat:sync) confirmation check --- .../src/components/ConnectorTreeComponent.tsx | 19 +++++++++++++++++-- frontend/src/locale/en.json | 1 + frontend/src/locale/es.json | 1 + frontend/src/locale/jp.json | 1 + frontend/src/locale/ru.json | 1 + frontend/src/locale/zh-TW.json | 1 + frontend/src/locale/zh.json | 1 + 7 files changed, 23 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/ConnectorTreeComponent.tsx b/frontend/src/components/ConnectorTreeComponent.tsx index 53900d0f..a191e4ab 100644 --- a/frontend/src/components/ConnectorTreeComponent.tsx +++ b/frontend/src/components/ConnectorTreeComponent.tsx @@ -3,8 +3,10 @@ import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; import { formatBytes } from '../utils/stringUtils'; import { selectToken } from '../preferences/preferenceSlice'; +import { ActiveState } from '../models/misc'; import Chunks from './Chunks'; import ContextMenu, { MenuOption } from './ContextMenu'; +import ConfirmationModal from '../modals/ConfirmationModal'; import userService from '../api/services/userService'; import FileIcon from '../assets/file.svg'; import FolderIcon from '../assets/folder.svg'; @@ -12,6 +14,7 @@ import ArrowLeft from '../assets/arrow-left.svg'; import ThreeDots from '../assets/three-dots.svg'; import EyeView from '../assets/eye-view.svg'; import SyncIcon from '../assets/sync.svg'; +import CheckmarkIcon from '../assets/checkMark2.svg'; import { useOutsideAlerter } from '../hooks'; interface FileNode { @@ -64,6 +67,7 @@ const ConnectorTreeComponent: React.FC = ({ const [syncProgress, setSyncProgress] = useState(0); const [sourceProvider, setSourceProvider] = useState(''); const [syncDone, setSyncDone] = useState(false); + const [syncConfirmationModal, setSyncConfirmationModal] = useState('INACTIVE'); useOutsideAlerter( searchDropdownRef, @@ -345,7 +349,7 @@ const ConnectorTreeComponent: React.FC = ({ {/* Sync button */}
)} + +
); }; diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 39e2bee7..b6e07e67 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -68,6 +68,7 @@ "private": "Private", "sync": "Sync", "syncing": "Syncing...", + "syncConfirmation": "Are you sure you want to sync \"{{sourceName}}\"? This will update the content with your cloud storage and may override any edits you made to individual chunks.", "syncFrequency": { "never": "Never", "daily": "Daily", diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index 1c8afa6c..4fb761f5 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -68,6 +68,7 @@ "private": "Privado", "sync": "Sincronizar", "syncing": "Sincronizando...", + "syncConfirmation": "¿Estás seguro de que deseas sincronizar \"{{sourceName}}\"? Esto actualizará el contenido con tu almacenamiento en la nube y puede anular cualquier edición que hayas realizado en fragmentos individuales.", "syncFrequency": { "never": "Nunca", "daily": "Diario", diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index ef29bd78..a805eecf 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -68,6 +68,7 @@ "private": "プライベート", "sync": "同期", "syncing": "同期中...", + "syncConfirmation": "\"{{sourceName}}\"を同期してもよろしいですか？これにより、コンテンツがクラウドストレージで更新され、個々のチャンクに加えた編集が上書きされる可能性があります。", "syncFrequency": { "never": "なし", "daily": "毎日", diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index a8506451..f2b2111e 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -68,6 +68,7 @@ "private": "Частный", "sync": "Синхронизация", "syncing": "Синхронизация...", + "syncConfirmation": "Вы уверены, что хотите синхронизировать \"{{sourceName}}\"? Это обновит содержимое с вашим облачным хранилищем и может перезаписать любые изменения, внесенные вами в отдельные фрагменты.", "syncFrequency": { "never": "Никогда", "daily": "Ежедневно", diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index 4f9c623b..a4247cf1 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -68,6 +68,7 @@ "private": "私人", "sync": "同步", "syncing": "同步中...", + "syncConfirmation": "您確定要同步 \"{{sourceName}}\" 嗎？這將使用您的雲端儲存更新內容，並可能覆蓋您對個別文本塊所做的任何編輯。", "syncFrequency": { "never": "從不", "daily": "每天", diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 014e8256..b03bd3e7 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -68,6 +68,7 @@ "private": "私有", "sync": "同步", "syncing": "同步中...", + "syncConfirmation": "您确定要同步 \"{{sourceName}}\" 吗？这将使用您的云存储更新内容，并可能覆盖您对单个文本块所做的任何编辑。", "syncFrequency": { "never": "从不", "daily": "每天", From cec8c72b4609f69a21bebd233c2662fa7bdd078b Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Wed, 10 Sep 2025 19:19:40 +0530 Subject: [PATCH 07/29] (refactor:uploads) YAGNI --- frontend/src/upload/Upload.tsx | 285 +++++++++----------------- frontend/src/upload/types/ingestor.ts | 175 +++------------- 2 files changed, 120 insertions(+), 340 deletions(-) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 3c42eca0..5d0d6a96 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -5,8 +5,7 @@ import { useDispatch, useSelector } from 'react-redux'; import userService from '../api/services/userService'; import { getSessionToken } from '../utils/providerUtils'; -import FileUpload from '../assets/file_upload.svg'; -import WebsiteCollect from '../assets/website_collect.svg'; + import Dropdown from '../components/Dropdown'; import Input from '../components/Input'; import ToggleSwitch from '../components/ToggleSwitch'; @@ -45,10 +44,9 @@ function Upload({ onSuccessfulUpload?: () => void; }) { const token = useSelector(selectToken); - const [docName, setDocName] = useState(receivedFile[0]?.name); - const [remoteName, setRemoteName] = useState(''); + const [files, setfiles] = useState(receivedFile); - const [activeTab, setActiveTab] = useState(renderTab); + const [activeTab, setActiveTab] = useState(true); const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); // File picker state @@ -59,11 +57,11 @@ function Upload({ const renderFormFields = () => { - const schema = IngestorFormSchemas[ingestor.type]; - if (!schema) return null; + if (!ingestor.type) return null; + const schema: FormField[] = IngestorFormSchemas[ingestor.type as IngestorType]; - const generalFields = schema.filter((field) => !field.advanced); - const advancedFields = schema.filter((field) => field.advanced); + const generalFields = schema.filter((field: FormField) => !field.advanced); + const advancedFields = schema.filter((field: FormField) => field.advanced); return (
@@ -184,30 +182,16 @@ function Upload({ className={`mt-2 text-base`} /> ); - case 'file_picker': - return ( - { - setSelectedFiles(selectedFileIds); - setSelectedFolders(selectedFolderIds); - }} - provider={ingestor.type} - token={token} - initialSelectedFiles={selectedFiles} - initialSelectedFolders={selectedFolders} - /> - ); case 'local_file_picker': return (
-
- +
+ Choose Files
-
+

Selected Files

@@ -230,21 +214,31 @@ function Upload({
); + case 'remote_file_picker': + return ( + { + setSelectedFiles(selectedFileIds); + setSelectedFolders(selectedFolderIds); + }} + provider={ingestor.type as unknown as string} + token={token} + initialSelectedFiles={selectedFiles} + initialSelectedFolders={selectedFolders} + /> + ); default: return null; } }; // New unified ingestor state - const [ingestor, setIngestor] = useState(() => { - const defaultType: IngestorType = 'crawler'; - const defaultConfig = IngestorDefaultConfigs[defaultType]; - return { - type: defaultType, - name: defaultConfig.name, - config: defaultConfig.config, - }; - }); + const [ingestor, setIngestor] = useState(() => ({ + type: null, + name: '', + config: {}, + })); const [progress, setProgress] = useState<{ type: 'UPLOAD' | 'TRAINING'; @@ -327,7 +321,7 @@ function Upload({ (progress?.percentage === 100 ? ( - -
-
- )} + - {activeTab === 'file' && ( - <> - setDocName(e.target.value)} - borderVariant="thin" - placeholder={t('modals.uploadDoc.name')} - labelBgClassName="bg-white dark:bg-charleston-green-2" - required={true} - /> -
- - - {t('modals.uploadDoc.choose')} - -
-

- {t('modals.uploadDoc.info')} -

-
-

- {t('modals.uploadDoc.uploadedFiles')} -

-
- {files.map((file) => ( -

- {file.name} -

- ))} - {files.length === 0 && ( -

- {t('none')} -

- )} -
-
- - )} - {activeTab === 'remote' && ( + + {activeTab && ( <> { - setRemoteName(e.target.value); - // Also update the ingestor name setIngestor((prevState) => ({ ...prevState, name: e.target.value, @@ -811,8 +719,8 @@ function Upload({ labelBgClassName="bg-white dark:bg-charleston-green-2" /> {renderFormFields()} - {IngestorFormSchemas[ingestor.type].some( - (field) => field.advanced, + {ingestor.type && IngestorFormSchemas[ingestor.type as IngestorType].some( + (field: FormField) => field.advanced, ) && ( - )} {activeTab && ( -
+ handleSearchChange(e.target.value)} + colorVariant="silver" + borderVariant="thin" + labelBgClassName="bg-white dark:bg-charleston-green-2" + leftIcon={Search} + />
diff --git a/frontend/src/components/Input.tsx b/frontend/src/components/Input.tsx index dbc517cb..b9f9d93a 100644 --- a/frontend/src/components/Input.tsx +++ b/frontend/src/components/Input.tsx @@ -16,6 +16,7 @@ const Input = ({ textSize = 'medium', children, labelBgClassName = 'bg-white dark:bg-raisin-black', + leftIcon, onChange, onPaste, onKeyDown, @@ -42,7 +43,7 @@ const Input = ({
{children} + {leftIcon && ( +
+ {leftIcon} +
+ )} {placeholder && (