From 8a7806ab2dcfd265fa22fa5d1513d6dc2ac500e3 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Tue, 15 Jul 2025 19:15:40 +0530 Subject: [PATCH] (feat:nested source view) file tree, chunks display --- frontend/src/api/endpoints.ts | 11 +- frontend/src/api/services/userService.ts | 5 +- frontend/src/assets/file.svg | 3 + frontend/src/assets/folder.svg | 3 + frontend/src/assets/outline-source.svg | 3 + frontend/src/components/DocumentChunks.tsx | 273 +++++++++++++++ frontend/src/components/FileTreeComponent.tsx | 199 +++++++---- frontend/src/models/misc.ts | 1 + frontend/src/settings/Documents.tsx | 312 ++---------------- 9 files changed, 452 insertions(+), 358 deletions(-) create mode 100644 frontend/src/assets/file.svg create mode 100644 frontend/src/assets/folder.svg create mode 100644 frontend/src/assets/outline-source.svg create mode 100644 frontend/src/components/DocumentChunks.tsx diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index bb98f02a..37c52c41 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -38,13 +38,20 @@ const endpoints = { UPDATE_TOOL_STATUS: '/api/update_tool_status', UPDATE_TOOL: '/api/update_tool', DELETE_TOOL: '/api/delete_tool', - GET_CHUNKS: (docId: string, page: number, per_page: number) => - `/api/get_chunks?id=${docId}&page=${page}&per_page=${per_page}`, + GET_CHUNKS: ( + docId: string, + page: number, + per_page: number, + path?: string, + ) => + `/api/get_chunks?id=${docId}&page=${page}&per_page=${per_page}${path ? `&path=${encodeURIComponent(path)}` : ''}`, ADD_CHUNK: '/api/add_chunk', DELETE_CHUNK: (docId: string, chunkId: string) => `/api/delete_chunk?id=${docId}&chunk_id=${chunkId}`, UPDATE_CHUNK: '/api/update_chunk', STORE_ATTACHMENT: '/api/store_attachment', + DIRECTORY_STRUCTURE: (docId: string) => + `/api/directory_structure?id=${docId}`, }, CONVERSATION: { ANSWER: '/api/answer', diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index ffb00a6b..43671657 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -86,8 +86,9 @@ const userService = { page: number, perPage: number, token: string | null, + path?: string, ): Promise => - apiClient.get(endpoints.USER.GET_CHUNKS(docId, page, perPage), token), + apiClient.get(endpoints.USER.GET_CHUNKS(docId, page, perPage, path), token), addChunk: (data: any, token: string | null): Promise => apiClient.post(endpoints.USER.ADD_CHUNK, data, token), deleteChunk: ( @@ -98,6 +99,8 @@ const userService = { apiClient.delete(endpoints.USER.DELETE_CHUNK(docId, chunkId), token), updateChunk: (data: any, token: string | null): Promise => apiClient.put(endpoints.USER.UPDATE_CHUNK, data, token), + getDirectoryStructure: (docId: string, token: string | null): Promise => + apiClient.get(endpoints.USER.DIRECTORY_STRUCTURE(docId), token), }; export default userService; diff --git a/frontend/src/assets/file.svg b/frontend/src/assets/file.svg new file mode 100644 index 00000000..7120a3c9 --- /dev/null +++ b/frontend/src/assets/file.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/assets/folder.svg b/frontend/src/assets/folder.svg new file mode 100644 index 00000000..2c2217f0 --- /dev/null +++ b/frontend/src/assets/folder.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/assets/outline-source.svg b/frontend/src/assets/outline-source.svg new file mode 100644 index 00000000..36b3aa6e --- /dev/null +++ b/frontend/src/assets/outline-source.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/components/DocumentChunks.tsx b/frontend/src/components/DocumentChunks.tsx new file mode 100644 index 00000000..94307bc0 --- /dev/null +++ b/frontend/src/components/DocumentChunks.tsx @@ -0,0 +1,273 @@ +import React, { useState, useEffect } from 'react'; +import { useSelector } from 'react-redux'; +import { useTranslation } from 'react-i18next'; +import { selectToken } from '../preferences/preferenceSlice'; +import { useDarkTheme, useLoaderState } from '../hooks'; +import userService from '../api/services/userService'; +import ArrowLeft from '../assets/arrow-left.svg'; +import NoFilesIcon from '../assets/no-files.svg'; +import NoFilesDarkIcon from '../assets/no-files-dark.svg'; +import Spinner from '../components/Spinner'; +import Input from '../components/Input'; +import ChunkModal from '../modals/ChunkModal'; +import { ActiveState } from '../models/misc'; +import { ChunkType } from '../settings/types'; + +interface DocumentChunksProps { + documentId: string; + documentName?: string; + handleGoBack: () => void; + showHeader?: boolean; + path?: string; +} + +const DocumentChunks: React.FC = ({ + documentId, + documentName, + handleGoBack, + showHeader = true, + path, +}) => { + const { t } = useTranslation(); + const token = useSelector(selectToken); + const [isDarkTheme] = useDarkTheme(); + const [paginatedChunks, setPaginatedChunks] = useState([]); + const [page, setPage] = useState(1); + const [perPage, setPerPage] = useState(5); + const [totalChunks, setTotalChunks] = useState(0); + const [loading, setLoading] = useLoaderState(true); + const [searchTerm, setSearchTerm] = useState(''); + const [addModal, setAddModal] = useState('INACTIVE'); + const [editModal, setEditModal] = useState<{ + state: ActiveState; + chunk: ChunkType | null; + }>({ state: 'INACTIVE', chunk: null }); + + const fetchChunks = () => { + setLoading(true); + try { + userService + .getDocumentChunks(documentId, page, perPage, token, path) + .then((response) => { + if (!response.ok) { + setLoading(false); + setPaginatedChunks([]); + throw new Error('Failed to fetch chunks data'); + } + return response.json(); + }) + .then((data) => { + setPage(data.page); + setPerPage(data.per_page); + setTotalChunks(data.total); + setPaginatedChunks(data.chunks); + setLoading(false); + }); + } catch (e) { + console.log(e); + setLoading(false); + } + }; + + const handleAddChunk = (title: string, text: string) => { + try { + userService + .addChunk( + { + id: documentId, + text: text, + metadata: { + title: title, + }, + }, + token, + ) + .then((response) => { + if (!response.ok) { + throw new Error('Failed to add chunk'); + } + fetchChunks(); + }); + } catch (e) { + console.log(e); + } + }; + + const handleUpdateChunk = (title: string, text: string, chunk: ChunkType) => { + try { + userService + .updateChunk( + { + id: documentId, + chunk_id: chunk.doc_id, + text: text, + metadata: { + title: title, + }, + }, + token, + ) + .then((response) => { + if (!response.ok) { + throw new Error('Failed to update chunk'); + } + fetchChunks(); + }); + } catch (e) { + console.log(e); + } + }; + + const handleDeleteChunk = (chunk: ChunkType) => { + try { + userService + .deleteChunk(documentId, chunk.doc_id, token) + .then((response) => { + if (!response.ok) { + throw new Error('Failed to delete chunk'); + } + setEditModal({ state: 'INACTIVE', chunk: null }); + fetchChunks(); + }); + } catch (e) { + console.log(e); + } + }; + + useEffect(() => { + fetchChunks(); + }, [page, perPage]); + + return ( +
+ {showHeader && ( +
+ +

{t('settings.documents.backToAll')}

+
+ )} +
+
+

{`${totalChunks} ${t('settings.documents.chunks')}`}

+ + { + setSearchTerm(e.target.value); + }} + borderVariant="thin" + /> +
+ +
+ {loading ? ( +
+
+ +
+
+ ) : ( +
+ {paginatedChunks.filter((chunk) => { + if (!chunk.metadata?.title) return true; + return chunk.metadata.title + .toLowerCase() + .includes(searchTerm.toLowerCase()); + }).length === 0 ? ( +
+ {t('settings.documents.noChunksAlt')} + {t('settings.documents.noChunks')} +
+ ) : ( + paginatedChunks + .filter((chunk) => { + if (!chunk.metadata?.title) return true; + return chunk.metadata.title + .toLowerCase() + .includes(searchTerm.toLowerCase()); + }) + .map((chunk, index) => ( +
+
+
+ +
+
+

+ {chunk.text} +

+
+
+
+ )) + )} +
+ )} + + + {editModal.chunk && ( + + setEditModal((prev) => ({ ...prev, state })) + } + handleSubmit={(title, text) => { + handleUpdateChunk(title, text, editModal.chunk as ChunkType); + }} + originalText={editModal.chunk?.text ?? ''} + originalTitle={editModal.chunk?.metadata?.title ?? ''} + handleDelete={() => { + handleDeleteChunk(editModal.chunk as ChunkType); + }} + /> + )} +
+ ); +}; + +export default DocumentChunks; diff --git a/frontend/src/components/FileTreeComponent.tsx b/frontend/src/components/FileTreeComponent.tsx index b106c106..d4252064 100644 --- a/frontend/src/components/FileTreeComponent.tsx +++ b/frontend/src/components/FileTreeComponent.tsx @@ -1,5 +1,8 @@ -import React, { useEffect, useState, useRef } from 'react'; +import React, { useState, useRef, useEffect } from 'react'; +import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; +import DocumentChunks from './DocumentChunks'; +import ContextMenu, { MenuOption } from './ContextMenu'; import userService from '../api/services/userService'; import FileIcon from '../assets/file.svg'; import FolderIcon from '../assets/folder.svg'; @@ -8,9 +11,6 @@ import ThreeDots from '../assets/three-dots.svg'; import EyeView from '../assets/eye-view.svg'; import OutlineSource from '../assets/outline-source.svg'; import Trash from '../assets/red-trash.svg'; -import Spinner from './Spinner'; -import { useTranslation } from 'react-i18next'; -import ContextMenu, { MenuOption } from './ContextMenu'; interface FileNode { type?: string; @@ -45,6 +45,18 @@ const FileTreeComponent: React.FC = ({ const menuRefs = useRef<{ [key: string]: React.RefObject; }>({}); + const [selectedFile, setSelectedFile] = useState<{ + id: string; + name: string; + } | null>(null); + + const handleFileClick = (fileName: string) => { + const fullPath = [...currentPath, fileName].join('/'); + setSelectedFile({ + id: fullPath, + name: fileName, + }); + }; useEffect(() => { const fetchDirectoryStructure = async () => { @@ -129,7 +141,11 @@ const FileTreeComponent: React.FC = ({ setActiveMenuId(itemId); }; - const getActionOptions = (name: string, isFile: boolean): MenuOption[] => { + const getActionOptions = ( + name: string, + isFile: boolean, + itemId: string, + ): MenuOption[] => { const options: MenuOption[] = []; if (isFile) { @@ -138,8 +154,7 @@ const FileTreeComponent: React.FC = ({ label: t('settings.documents.view'), onClick: (event: React.SyntheticEvent) => { event.stopPropagation(); - console.log('View file:', name); - // View file action will be implemented later + handleFileClick(name); }, iconWidth: 18, iconHeight: 18, @@ -195,32 +210,89 @@ const FileTreeComponent: React.FC = ({ ); }; + + const calculateDirectoryStats = ( + structure: DirectoryStructure, + ): { totalSize: number; totalTokens: number } => { + let totalSize = 0; + let totalTokens = 0; + + Object.entries(structure).forEach(([_, node]) => { + if (node.type) { + // It's a file + totalSize += node.size_bytes || 0; + totalTokens += node.token_count || 0; + } else { + // It's a directory, recurse + const stats = calculateDirectoryStats(node); + totalSize += stats.totalSize; + totalTokens += stats.totalTokens; + } + }); + + return { totalSize, totalTokens }; + }; + const renderFileTree = (structure: DirectoryStructure): React.ReactNode[] => { + // Separate directories and files const entries = Object.entries(structure); const directories = entries.filter(([_, node]) => !node.type); const files = entries.filter(([_, node]) => node.type); + // Create parent directory row + const parentRow = + currentPath.length > 0 + ? [ + + +
+ Parent folder + .. +
+ + - + - + + , + ] + : []; + + // Render directories first, then files return [ + ...parentRow, ...directories.map(([name, node]) => { const itemId = `dir-${name}`; const menuRef = getMenuRef(itemId); + const dirStats = calculateDirectoryStats(node as DirectoryStructure); return ( navigateToDirectory(name)} > -
navigateToDirectory(name)} - > +
Folder - {name} + {name}
- - - - + + {dirStats.totalTokens > 0 + ? dirStats.totalTokens.toLocaleString() + : '-'} + + + {dirStats.totalSize > 0 ? formatBytes(dirStats.totalSize) : '-'} +
-

{t('settings.documents.backToAll')}

-
-
-
-

{`${totalChunks} ${t('settings.documents.chunks')}`}

- - { - setSearchTerm(e.target.value); - }} - borderVariant="thin" - /> -
- -
- {loading ? ( -
-
- -
-
- ) : ( -
- {paginatedChunks.filter((chunk) => { - if (!chunk.metadata?.title) return true; - return chunk.metadata.title - .toLowerCase() - .includes(searchTerm.toLowerCase()); - }).length === 0 ? ( -
- {t('settings.documents.noChunksAlt')} - {t('settings.documents.noChunks')} -
- ) : ( - paginatedChunks - .filter((chunk) => { - if (!chunk.metadata?.title) return true; - return chunk.metadata.title - .toLowerCase() - .includes(searchTerm.toLowerCase()); - }) - .map((chunk, index) => ( -
-
-
- -
-
-

- {chunk.metadata?.title ?? 'Untitled'} -

-

- {chunk.text} -

-
-
-
- )) - )} -
- )} - {!loading && - paginatedChunks.filter((chunk) => { - if (!chunk.metadata?.title) return true; - return chunk.metadata.title - .toLowerCase() - .includes(searchTerm.toLowerCase()); - }).length !== 0 && ( -
- { - setPage(page); - }} - onRowsPerPageChange={(rows) => { - setPerPage(rows); - setPage(1); - }} - /> -
- )} - - {editModal.chunk && ( - - setEditModal((prev) => ({ ...prev, state })) - } - handleSubmit={(title, text) => { - handleUpdateChunk(title, text, editModal.chunk as ChunkType); - }} - originalText={editModal.chunk?.text ?? ''} - originalTitle={editModal.chunk?.metadata?.title ?? ''} - handleDelete={() => { - handleDeleteChunk(editModal.chunk as ChunkType); - }} - /> - )} -
- ); -}