From 5debb482651b11e77d3ecf81696b5aedf43cd00b Mon Sep 17 00:00:00 2001 From: fadingNA Date: Fri, 8 Nov 2024 11:02:13 -0500 Subject: [PATCH] Paginated With MongoDB / Create New Endpoint change routes /combine name, add route /api/source/paginated add new endpoint source/paginated fixing table responsive create new function to handling api/source/paginated --- application/api/user/routes.py | 96 +++++++++++++++++------ frontend/src/Navigation.tsx | 5 +- frontend/src/api/endpoints.ts | 3 +- frontend/src/api/services/userService.ts | 13 +-- frontend/src/hooks/useDefaultDocument.ts | 11 +-- frontend/src/index.css | 4 +- frontend/src/models/misc.ts | 2 +- frontend/src/preferences/preferenceApi.ts | 76 +++++++++--------- frontend/src/settings/Documents.tsx | 40 ++++++---- frontend/src/settings/index.tsx | 7 +- frontend/src/upload/Upload.tsx | 27 ++++--- 11 files changed, 173 insertions(+), 111 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index e5ad68ae..7d44ce9b 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -7,7 +7,7 @@ import math from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef from bson.objectid import ObjectId -from flask import Blueprint, jsonify, make_response, request +from flask import Blueprint, jsonify, make_response, request, redirect from flask_restx import inputs, fields, Namespace, Resource from werkzeug.utils import secure_filename @@ -430,14 +430,63 @@ class TaskStatus(Resource): @user_ns.route("/api/combine") +class RedirectToSources(Resource): + @api.doc( + description="Redirects /api/combine to /api/sources for backward compatibility" + ) + def get(self): + return redirect("/api/sources", code=301) + + +@user_ns.route("/api/sources/paginated") +class PaginatedSources(Resource): + @api.doc(description="Get document with pagination, sorting and filtering") + def get(self): + user = "local" + sort_field = request.args.get("sort", "date") # Default to 'date' + sort_order = request.args.get("order", "desc") # Default to 'desc' + page = int(request.args.get("page", 1)) # Default to 1 + rows_per_page = int(request.args.get("rows", 10)) # Default to 10 + + # Prepare + query = {"user": user} + total_documents = sources_collection.count_documents(query) + total_pages = max(1, math.ceil(total_documents / rows_per_page)) + sort_order = 1 if sort_order == "asc" else -1 + skip = (page - 1) * rows_per_page + + try: + documents = ( + sources_collection.find(query) + .sort(sort_field, sort_order) + .skip(skip) + .limit(rows_per_page) + ) + + paginated_docs = [] + for doc in documents: + doc["_id"] = str(doc["_id"]) + paginated_docs.append(doc) + + response = { + "total": total_documents, + "totalPages": total_pages, + "currentPage": page, + "paginated": paginated_docs, + } + return make_response(jsonify(response), 200) + + except Exception as err: + return make_response(jsonify({"success": False, "error": str(err)}), 400) + + +@user_ns.route("/api/sources") class CombinedJson(Resource): @api.doc(description="Provide JSON file with combined available indexes") def get(self): user = "local" - sort_field = request.args.get('sort', 'date') # Default to 'date' - sort_order = request.args.get('order', "desc") # Default to 'desc' - page = int(request.args.get('page', 1)) # Default to 1 - rows_per_page = int(request.args.get('rows', 10)) # Default to 10 + sort_field = request.args.get("sort", "date") # Default to 'date' + sort_order = request.args.get("order", "desc") # Default to 'desc' data = [ { "name": "default", @@ -450,7 +499,9 @@ class CombinedJson(Resource): ] try: - for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1): + for index in sources_collection.find({"user": user}).sort( + sort_field, 1 if sort_order == "asc" else -1 + ): data.append( { "id": str(index["_id"]), @@ -488,23 +539,11 @@ class CombinedJson(Resource): "retriever": "brave_search", } ) - total_documents = len(data) - total_pages = max(1, math.ceil(total_documents / rows_per_page)) - - first_index = (page - 1) * rows_per_page - last_index = first_index + rows_per_page - paginated_docs = data[first_index:last_index] except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - - response = { - "paginated_docs": paginated_docs, - "totalDocuments": total_documents, - "totalPages": total_pages, - "documents":data - } - return make_response(jsonify(response), 200) + + return make_response(jsonify(data), 200) @user_ns.route("/api/docs_check") @@ -1690,7 +1729,9 @@ class TextToSpeech(Resource): tts_model = api.model( "TextToSpeechModel", { - "text": fields.String(required=True, description="Text to be synthesized as audio"), + "text": fields.String( + required=True, description="Text to be synthesized as audio" + ), }, ) @@ -1702,8 +1743,15 @@ class TextToSpeech(Resource): try: tts_instance = GoogleTTS() audio_base64, detected_language = tts_instance.text_to_speech(text) - return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200) + return make_response( + jsonify( + { + "success": True, + "audio_base64": audio_base64, + "lang": detected_language, + } + ), + 200, + ) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - - diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index 3591469b..72c0d57a 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -145,7 +145,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { dispatch(setSourceDocs(updatedDocs)); dispatch( setSelectedDocs( - updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'), + Array.isArray(updatedDocs) && + updatedDocs?.find( + (doc: Doc) => doc.name.toLowerCase() === 'default', + ), ), ); }) diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index 84674049..4e7112d0 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -1,7 +1,8 @@ const endpoints = { USER: { - DOCS: '/api/combine', + DOCS: '/api/sources', DOCS_CHECK: '/api/docs_check', + DOCS_PAGINATED: '/api/sources/paginated', API_KEYS: '/api/get_api_keys', CREATE_API_KEY: '/api/create_api_key', DELETE_API_KEY: '/api/delete_api_key', diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index bebadca5..82f32244 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -2,15 +2,10 @@ import apiClient from '../client'; import endpoints from '../endpoints'; const userService = { - getDocs: ( - sort: string, - order: string, - pageNumber: number, - rowsPerPage: number, - ): Promise => - apiClient.get( - `${endpoints.USER.DOCS}?sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`, - ), + getDocs: (sort: string, order: string): Promise => + apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`), + getDocsWithPagination: (query: string): Promise => + apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`), checkDocs: (data: any): Promise => apiClient.post(endpoints.USER.DOCS_CHECK, data), getAPIKeys: (): Promise => apiClient.get(endpoints.USER.API_KEYS), diff --git a/frontend/src/hooks/useDefaultDocument.ts b/frontend/src/hooks/useDefaultDocument.ts index 37374ce0..7f4b9812 100644 --- a/frontend/src/hooks/useDefaultDocument.ts +++ b/frontend/src/hooks/useDefaultDocument.ts @@ -17,11 +17,12 @@ export default function useDefaultDocument() { getDocs().then((data) => { dispatch(setSourceDocs(data)); if (!selectedDoc) - data?.forEach((doc: Doc) => { - if (doc.model && doc.name === 'default') { - dispatch(setSelectedDocs(doc)); - } - }); + Array.isArray(data) && + data?.forEach((doc: Doc) => { + if (doc.model && doc.name === 'default') { + dispatch(setSelectedDocs(doc)); + } + }); }); }; diff --git a/frontend/src/index.css b/frontend/src/index.css index 090a31ca..e8e9c9bd 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -50,11 +50,11 @@ body.dark { @layer components { .table-default { - @apply block w-max mx-auto table-fixed content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray; + @apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray; } .table-default th { - @apply p-4 w-[250px] font-normal text-gray-400; /* Remove border-r */ + @apply p-4 w-1/4 font-normal text-gray-400 text-nowrap; /* Remove border-r */ } .table-default th:last-child { diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts index 5478722c..46af487c 100644 --- a/frontend/src/models/misc.ts +++ b/frontend/src/models/misc.ts @@ -18,6 +18,7 @@ export type GetDocsResponse = { docs: Doc[]; totalDocuments: number; totalPages: number; + nextCursor: string; }; export type PromptProps = { @@ -28,7 +29,6 @@ export type PromptProps = { }; export type DocumentsProps = { - documents: Doc[] | null; handleDeleteDocument: (index: number, document: Doc) => void; }; diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index e77c9e3b..e3d22c02 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -4,49 +4,53 @@ import { Doc, GetDocsResponse } from '../models/misc'; //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later. export async function getDocs( - sort?: string, - order?: string, - pageNumber?: number, - rowsPerPage?: number, - withPagination?: true, -): Promise; + sort = 'date', + order = 'desc', +): Promise { + try { + const response = await userService.getDocs(sort, order); + const data = await response.json(); -export async function getDocs( + const docs: Doc[] = []; + console.log(data); + data.forEach((doc: object) => { + docs.push(doc as Doc); + }); + + return docs; + } catch (error) { + console.log(error); + return null; + } +} + +export async function getDocsWithPagination( sort = 'date', order = 'desc', pageNumber = 1, - rowsPerPage = 5, - withPagination = false, -): Promise { + rowsPerPage = 10, +): Promise { try { - const response = await userService.getDocs( - sort, - order, - pageNumber, - rowsPerPage, - ); + const query = `sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`; + const response = await userService.getDocsWithPagination(query); const data = await response.json(); - console.log(data); - if (withPagination) { - const docs: Doc[] = []; - Array.isArray(data.paginated_docs) && - data.paginated_docs.forEach((doc: object) => { - docs.push(doc as Doc); - }); - const totalDocuments = data.totalDocuments || 0; - const totalPages = data.totalPages || 0; - console.log(`totalDocuments: ${totalDocuments}`); - console.log(`totalPages: ${totalPages}`); - return { docs, totalDocuments, totalPages }; - } else { - const docs: Doc[] = []; - Array.isArray(data.documents) && - data.documents.forEach((doc: object) => { - docs.push(doc as Doc); - }); - return docs; - } + const docs: Doc[] = []; + console.log(`data: ${data}`); + Array.isArray(data.paginated) && + data.paginated.forEach((doc: Doc) => { + docs.push(doc as Doc); + }); + console.log(`total: ${data.total}`); + console.log(`totalPages: ${data.totalPages}`); + console.log(`cursor: ${data.nextCursor}`); + console.log(`currentPage: ${data.currentPage}`); + return { + docs: docs, + totalDocuments: data.total, + totalPages: data.totalPages, + nextCursor: data.nextCursor, + }; } catch (error) { console.log(error); return null; diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index fc21a8f1..8a35ca7d 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -10,7 +10,7 @@ import caretSort from '../assets/caret-sort.svg'; import DropdownMenu from '../components/DropdownMenu'; import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported import SkeletonLoader from '../components/SkeletonLoader'; -import { getDocs } from '../preferences/preferenceApi'; +import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi'; import { setSourceDocs } from '../preferences/preferenceSlice'; import Input from '../components/Input'; import Upload from '../upload/Upload'; // Import the Upload component @@ -33,13 +33,9 @@ const formatTokens = (tokens: number): string => { } }; -const Documents: React.FC = ({ - documents, - handleDeleteDocument, -}) => { +const Documents: React.FC = ({ handleDeleteDocument }) => { const { t } = useTranslation(); const dispatch = useDispatch(); - // State for search input const [searchTerm, setSearchTerm] = useState(''); // State for modal: active/inactive @@ -60,7 +56,6 @@ const Documents: React.FC = ({ ); // State for documents const currentDocuments = filteredDocuments ?? []; - const syncOptions = [ { label: 'Never', value: 'never' }, { label: 'Daily', value: 'daily' }, @@ -73,9 +68,9 @@ const Documents: React.FC = ({ pageNumber?: number, rows?: number, ) => { - console.log(`field: ${field}, pageNumber: ${pageNumber}, rows: ${rows}`); const page = pageNumber ?? currentPage; const rowsPerPg = rows ?? rowsPerPage; + if (field !== undefined) { if (field === sortField) { // Toggle sort order @@ -86,9 +81,9 @@ const Documents: React.FC = ({ setSortOrder('desc'); } } - getDocs(sortField, sortOrder, page, rowsPerPg, true) + getDocsWithPagination(sortField, sortOrder, page, rowsPerPg) .then((data) => { - console.log(data); + console.log('Data received from getDocsWithPagination:', data); dispatch(setSourceDocs(data ? data.docs : [])); setFetchedDocuments(data ? data.docs : []); setTotalPages(data ? data.totalPages : 0); @@ -99,6 +94,7 @@ const Documents: React.FC = ({ setLoading(false); }); }; + const handleManageSync = (doc: Doc, sync_frequency: string) => { setLoading(true); userService @@ -114,9 +110,13 @@ const Documents: React.FC = ({ setLoading(false); }); }; + useEffect(() => { - refreshDocs(sortField, currentPage, rowsPerPage); - }, []); + if (modalState === 'INACTIVE') { + refreshDocs(sortField, currentPage, rowsPerPage); + } + }, [modalState, sortField, currentPage, rowsPerPage]); + return (
@@ -190,7 +190,7 @@ const Documents: React.FC = ({ )} {Array.isArray(currentDocuments) && currentDocuments.map((document, index) => ( - + {document.name} {document.date} @@ -248,18 +248,24 @@ const Documents: React.FC = ({
)}
+ {/* Pagination component with props: + # Note: Every time the page changes, + the refreshDocs function is called with the updated page number and rows per page. + and reset cursor paginated query parameter to undefined. + */} { setCurrentPage(page); - refreshDocs(undefined, page, rowsPerPage); + refreshDocs(sortField, page, rowsPerPage); // Pass `true` to reset lastID if not using cursor }} onRowsPerPageChange={(rows) => { + console.log('Pagination - Rows per Page Change:', rows); setRowsPerPage(rows); - setCurrentPage(1); - refreshDocs(undefined, 1, rows); + setCurrentPage(1); // Reset to page 1 on rows per page change + refreshDocs(sortField, 1, rows); // Reset lastID for fresh pagination }} /> @@ -267,7 +273,7 @@ const Documents: React.FC = ({ }; Documents.propTypes = { - documents: PropTypes.array.isRequired, + //documents: PropTypes.array.isRequired, handleDeleteDocument: PropTypes.func.isRequired, }; diff --git a/frontend/src/settings/index.tsx b/frontend/src/settings/index.tsx index ea3d4428..24ea2681 100644 --- a/frontend/src/settings/index.tsx +++ b/frontend/src/settings/index.tsx @@ -70,12 +70,7 @@ export default function Settings() { case t('settings.general.label'): return ; case t('settings.documents.label'): - return ( - - ); + return ; case 'Widgets': return ( d.type?.toLowerCase() === 'local'), + Array.isArray(data) && + data?.find( + (d: Doc) => d.type?.toLowerCase() === 'local', + ), ), ); }); @@ -182,15 +185,21 @@ function Upload({ getDocs().then((data) => { dispatch(setSourceDocs(data)); const docIds = new Set( - sourceDocs?.map((doc: Doc) => (doc.id ? doc.id : null)), + (Array.isArray(sourceDocs) && + sourceDocs?.map((doc: Doc) => + doc.id ? doc.id : null, + )) || + [], ); - data?.map((updatedDoc: Doc) => { - if (updatedDoc.id && !docIds.has(updatedDoc.id)) { - //select the doc not present in the intersection of current Docs and fetched data - dispatch(setSelectedDocs(updatedDoc)); - return; - } - }); + if (data && Array.isArray(data.docs)) { + data.docs.map((updatedDoc: Doc) => { + if (updatedDoc.id && !docIds.has(updatedDoc.id)) { + // Select the doc not present in the intersection of current Docs and fetched data + dispatch(setSelectedDocs(updatedDoc)); + return; + } + }); + } }); setProgress( (progress) =>