Paginated With MongoDB / Create New Endpoint

change routes /combine name, add route /api/source/paginated add new endpoint source/paginated fixing table responsive create new function to handling api/source/paginated
2025-11-29 08:33:20 +00:00 · 2024-11-08 11:02:13 -05:00
parent 4429755c09
commit 5debb48265
11 changed files with 173 additions and 111 deletions
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -7,7 +7,7 @@ import math
 from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
-from flask import Blueprint, jsonify, make_response, request
+from flask import Blueprint, jsonify, make_response, request, redirect
 from flask_restx import inputs, fields, Namespace, Resource
 from werkzeug.utils import secure_filename

@@ -430,14 +430,63 @@ class TaskStatus(Resource):


@user_ns.route("/api/combine")
+class RedirectToSources(Resource):
+    @api.doc(
+        description="Redirects /api/combine to /api/sources for backward compatibility"
+    )
+    def get(self):
+        return redirect("/api/sources", code=301)
+
+
+@user_ns.route("/api/sources/paginated")
+class PaginatedSources(Resource):
+    @api.doc(description="Get document with pagination, sorting and filtering")
+    def get(self):
+        user = "local"
+        sort_field = request.args.get("sort", "date")  # Default to 'date'
+        sort_order = request.args.get("order", "desc")  # Default to 'desc'
+        page = int(request.args.get("page", 1))  # Default to 1
+        rows_per_page = int(request.args.get("rows", 10))  # Default to 10
+
+        # Prepare
+        query = {"user": user}
+        total_documents = sources_collection.count_documents(query)
+        total_pages = max(1, math.ceil(total_documents / rows_per_page))
+        sort_order = 1 if sort_order == "asc" else -1
+        skip = (page - 1) * rows_per_page
+
+        try:
+            documents = (
+                sources_collection.find(query)
+                .sort(sort_field, sort_order)
+                .skip(skip)
+                .limit(rows_per_page)
+            )
+
+            paginated_docs = []
+            for doc in documents:
+                doc["_id"] = str(doc["_id"])
+                paginated_docs.append(doc)
+
+            response = {
+                "total": total_documents,
+                "totalPages": total_pages,
+                "currentPage": page,
+                "paginated": paginated_docs,
+            }
+            return make_response(jsonify(response), 200)
+
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+
+@user_ns.route("/api/sources")
 class CombinedJson(Resource):
    @api.doc(description="Provide JSON file with combined available indexes")
    def get(self):
        user = "local"
-        sort_field = request.args.get('sort', 'date')  # Default to 'date'
-        sort_order = request.args.get('order', "desc")  # Default to 'desc'
-        page = int(request.args.get('page', 1)) # Default to 1
-        rows_per_page = int(request.args.get('rows', 10)) # Default to 10
+        sort_field = request.args.get("sort", "date")  # Default to 'date'
+        sort_order = request.args.get("order", "desc")  # Default to 'desc'
        data = [
            {
                "name": "default",
@@ -450,7 +499,9 @@ class CombinedJson(Resource):
        ]

        try:
-            for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1):
+            for index in sources_collection.find({"user": user}).sort(
+                sort_field, 1 if sort_order == "asc" else -1
+            ):
                data.append(
                    {
                        "id": str(index["_id"]),
@@ -488,23 +539,11 @@ class CombinedJson(Resource):
                        "retriever": "brave_search",
                    }
                )
-            total_documents = len(data)
-            total_pages = max(1, math.ceil(total_documents / rows_per_page))
-
-            first_index = (page - 1) * rows_per_page
-            last_index = first_index + rows_per_page
-            paginated_docs = data[first_index:last_index]

        except Exception as err:
            return make_response(jsonify({"success": False, "error": str(err)}), 400)
-            
-        response = {
-            "paginated_docs": paginated_docs,
-            "totalDocuments": total_documents,
-            "totalPages": total_pages,
-            "documents":data
-        }
-        return make_response(jsonify(response), 200)
+
+        return make_response(jsonify(data), 200)


@user_ns.route("/api/docs_check")
@@ -1690,7 +1729,9 @@ class TextToSpeech(Resource):
    tts_model = api.model(
        "TextToSpeechModel",
        {
-            "text": fields.String(required=True, description="Text to be synthesized as audio"),
+            "text": fields.String(
+                required=True, description="Text to be synthesized as audio"
+            ),
        },
    )

@@ -1702,8 +1743,15 @@ class TextToSpeech(Resource):
        try:
            tts_instance = GoogleTTS()
            audio_base64, detected_language = tts_instance.text_to_speech(text)
-            return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "audio_base64": audio_base64,
+                        "lang": detected_language,
+                    }
+                ),
+                200,
+            )
        except Exception as err:
            return make_response(jsonify({"success": False, "error": str(err)}), 400)
-
-
--- a/frontend/src/Navigation.tsx
+++ b/frontend/src/Navigation.tsx
@@ -145,7 +145,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
        dispatch(setSourceDocs(updatedDocs));
        dispatch(
          setSelectedDocs(
-            updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'),
+            Array.isArray(updatedDocs) &&
+              updatedDocs?.find(
+                (doc: Doc) => doc.name.toLowerCase() === 'default',
+              ),
          ),
        );
      })
--- a/frontend/src/api/endpoints.ts
+++ b/frontend/src/api/endpoints.ts
@@ -1,7 +1,8 @@
 const endpoints = {
  USER: {
-    DOCS: '/api/combine',
+    DOCS: '/api/sources',
    DOCS_CHECK: '/api/docs_check',
+    DOCS_PAGINATED: '/api/sources/paginated',
    API_KEYS: '/api/get_api_keys',
    CREATE_API_KEY: '/api/create_api_key',
    DELETE_API_KEY: '/api/delete_api_key',
--- a/frontend/src/api/services/userService.ts
+++ b/frontend/src/api/services/userService.ts
@@ -2,15 +2,10 @@ import apiClient from '../client';
 import endpoints from '../endpoints';

 const userService = {
-  getDocs: (
-    sort: string,
-    order: string,
-    pageNumber: number,
-    rowsPerPage: number,
-  ): Promise<any> =>
-    apiClient.get(
-      `${endpoints.USER.DOCS}?sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`,
-    ),
+  getDocs: (sort: string, order: string): Promise<any> =>
+    apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`),
+  getDocsWithPagination: (query: string): Promise<any> =>
+    apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`),
  checkDocs: (data: any): Promise<any> =>
    apiClient.post(endpoints.USER.DOCS_CHECK, data),
  getAPIKeys: (): Promise<any> => apiClient.get(endpoints.USER.API_KEYS),
--- a/frontend/src/hooks/useDefaultDocument.ts
+++ b/frontend/src/hooks/useDefaultDocument.ts
@@ -17,11 +17,12 @@ export default function useDefaultDocument() {
    getDocs().then((data) => {
      dispatch(setSourceDocs(data));
      if (!selectedDoc)
-        data?.forEach((doc: Doc) => {
-          if (doc.model && doc.name === 'default') {
-            dispatch(setSelectedDocs(doc));
-          }
-        });
+        Array.isArray(data) &&
+          data?.forEach((doc: Doc) => {
+            if (doc.model && doc.name === 'default') {
+              dispatch(setSelectedDocs(doc));
+            }
+          });
    });
  };

--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -50,11 +50,11 @@ body.dark {

@layer components {
  .table-default {
-    @apply block w-max mx-auto table-fixed content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
+    @apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
  }

  .table-default th {
-    @apply p-4 w-[250px] font-normal text-gray-400; /* Remove border-r */
+    @apply p-4 w-1/4 font-normal text-gray-400 text-nowrap; /* Remove border-r */
  }

  .table-default th:last-child {
--- a/frontend/src/models/misc.ts
+++ b/frontend/src/models/misc.ts
@@ -18,6 +18,7 @@ export type GetDocsResponse = {
  docs: Doc[];
  totalDocuments: number;
  totalPages: number;
+  nextCursor: string;
 };

 export type PromptProps = {
@@ -28,7 +29,6 @@ export type PromptProps = {
 };

 export type DocumentsProps = {
-  documents: Doc[] | null;
  handleDeleteDocument: (index: number, document: Doc) => void;
 };

--- a/frontend/src/preferences/preferenceApi.ts
+++ b/frontend/src/preferences/preferenceApi.ts
@@ -4,49 +4,53 @@ import { Doc, GetDocsResponse } from '../models/misc';

 //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
 export async function getDocs(
-  sort?: string,
-  order?: string,
-  pageNumber?: number,
-  rowsPerPage?: number,
-  withPagination?: true,
-): Promise<GetDocsResponse | null>;
+  sort = 'date',
+  order = 'desc',
+): Promise<Doc[] | null> {
+  try {
+    const response = await userService.getDocs(sort, order);
+    const data = await response.json();

-export async function getDocs(
+    const docs: Doc[] = [];
+    console.log(data);
+    data.forEach((doc: object) => {
+      docs.push(doc as Doc);
+    });
+
+    return docs;
+  } catch (error) {
+    console.log(error);
+    return null;
+  }
+}
+
+export async function getDocsWithPagination(
  sort = 'date',
  order = 'desc',
  pageNumber = 1,
-  rowsPerPage = 5,
-  withPagination = false,
-): Promise<Doc[] | GetDocsResponse | null> {
+  rowsPerPage = 10,
+): Promise<GetDocsResponse | null> {
  try {
-    const response = await userService.getDocs(
-      sort,
-      order,
-      pageNumber,
-      rowsPerPage,
-    );
+    const query = `sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`;
+    const response = await userService.getDocsWithPagination(query);
    const data = await response.json();
-    console.log(data);
-    if (withPagination) {
-      const docs: Doc[] = [];
-      Array.isArray(data.paginated_docs) &&
-        data.paginated_docs.forEach((doc: object) => {
-          docs.push(doc as Doc);
-        });

-      const totalDocuments = data.totalDocuments || 0;
-      const totalPages = data.totalPages || 0;
-      console.log(`totalDocuments: ${totalDocuments}`);
-      console.log(`totalPages: ${totalPages}`);
-      return { docs, totalDocuments, totalPages };
-    } else {
-      const docs: Doc[] = [];
-      Array.isArray(data.documents) &&
-        data.documents.forEach((doc: object) => {
-          docs.push(doc as Doc);
-        });
-      return docs;
-    }
+    const docs: Doc[] = [];
+    console.log(`data: ${data}`);
+    Array.isArray(data.paginated) &&
+      data.paginated.forEach((doc: Doc) => {
+        docs.push(doc as Doc);
+      });
+    console.log(`total: ${data.total}`);
+    console.log(`totalPages: ${data.totalPages}`);
+    console.log(`cursor: ${data.nextCursor}`);
+    console.log(`currentPage: ${data.currentPage}`);
+    return {
+      docs: docs,
+      totalDocuments: data.total,
+      totalPages: data.totalPages,
+      nextCursor: data.nextCursor,
+    };
  } catch (error) {
    console.log(error);
    return null;
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -10,7 +10,7 @@ import caretSort from '../assets/caret-sort.svg';
 import DropdownMenu from '../components/DropdownMenu';
 import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported
 import SkeletonLoader from '../components/SkeletonLoader';
-import { getDocs } from '../preferences/preferenceApi';
+import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi';
 import { setSourceDocs } from '../preferences/preferenceSlice';
 import Input from '../components/Input';
 import Upload from '../upload/Upload'; // Import the Upload component
@@ -33,13 +33,9 @@ const formatTokens = (tokens: number): string => {
  }
 };

-const Documents: React.FC<DocumentsProps> = ({
-  documents,
-  handleDeleteDocument,
-}) => {
+const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
  const { t } = useTranslation();
  const dispatch = useDispatch();
-
  // State for search input
  const [searchTerm, setSearchTerm] = useState('');
  // State for modal: active/inactive
@@ -60,7 +56,6 @@ const Documents: React.FC<DocumentsProps> = ({
  );
  // State for documents
  const currentDocuments = filteredDocuments ?? [];
-
  const syncOptions = [
    { label: 'Never', value: 'never' },
    { label: 'Daily', value: 'daily' },
@@ -73,9 +68,9 @@ const Documents: React.FC<DocumentsProps> = ({
    pageNumber?: number,
    rows?: number,
  ) => {
-    console.log(`field: ${field}, pageNumber: ${pageNumber}, rows: ${rows}`);
    const page = pageNumber ?? currentPage;
    const rowsPerPg = rows ?? rowsPerPage;
+
    if (field !== undefined) {
      if (field === sortField) {
        // Toggle sort order
@@ -86,9 +81,9 @@ const Documents: React.FC<DocumentsProps> = ({
        setSortOrder('desc');
      }
    }
-    getDocs(sortField, sortOrder, page, rowsPerPg, true)
+    getDocsWithPagination(sortField, sortOrder, page, rowsPerPg)
      .then((data) => {
-        console.log(data);
+        console.log('Data received from getDocsWithPagination:', data);
        dispatch(setSourceDocs(data ? data.docs : []));
        setFetchedDocuments(data ? data.docs : []);
        setTotalPages(data ? data.totalPages : 0);
@@ -99,6 +94,7 @@ const Documents: React.FC<DocumentsProps> = ({
        setLoading(false);
      });
  };
+
  const handleManageSync = (doc: Doc, sync_frequency: string) => {
    setLoading(true);
    userService
@@ -114,9 +110,13 @@ const Documents: React.FC<DocumentsProps> = ({
        setLoading(false);
      });
  };
+
  useEffect(() => {
-    refreshDocs(sortField, currentPage, rowsPerPage);
-  }, []);
+    if (modalState === 'INACTIVE') {
+      refreshDocs(sortField, currentPage, rowsPerPage);
+    }
+  }, [modalState, sortField, currentPage, rowsPerPage]);
+
  return (
    <div className="mt-8">
      <div className="flex flex-col relative">
@@ -190,7 +190,7 @@ const Documents: React.FC<DocumentsProps> = ({
                )}
                {Array.isArray(currentDocuments) &&
                  currentDocuments.map((document, index) => (
-                    <tr key={index}>
+                    <tr key={index} className="text-nowrap font-normal">
                      <td>{document.name}</td>
                      <td>{document.date}</td>
                      <td>
@@ -248,18 +248,24 @@ const Documents: React.FC<DocumentsProps> = ({
          </div>
        )}
      </div>
+      {/* Pagination component with props:
+      # Note: Every time the page changes, 
+      the refreshDocs function is called with the updated page number and rows per page.
+      and reset cursor paginated query parameter to undefined.
+      */}
      <Pagination
        currentPage={currentPage}
        totalPages={totalPages}
        rowsPerPage={rowsPerPage}
        onPageChange={(page) => {
          setCurrentPage(page);
-          refreshDocs(undefined, page, rowsPerPage);
+          refreshDocs(sortField, page, rowsPerPage); // Pass `true` to reset lastID if not using cursor
        }}
        onRowsPerPageChange={(rows) => {
+          console.log('Pagination - Rows per Page Change:', rows);
          setRowsPerPage(rows);
-          setCurrentPage(1);
-          refreshDocs(undefined, 1, rows);
+          setCurrentPage(1); // Reset to page 1 on rows per page change
+          refreshDocs(sortField, 1, rows); // Reset lastID for fresh pagination
        }}
      />
    </div>
@@ -267,7 +273,7 @@ const Documents: React.FC<DocumentsProps> = ({
 };

 Documents.propTypes = {
-  documents: PropTypes.array.isRequired,
+  //documents: PropTypes.array.isRequired,
  handleDeleteDocument: PropTypes.func.isRequired,
 };

--- a/frontend/src/settings/index.tsx
+++ b/frontend/src/settings/index.tsx
@@ -70,12 +70,7 @@ export default function Settings() {
      case t('settings.general.label'):
        return <General />;
      case t('settings.documents.label'):
-        return (
-          <Documents
-            documents={documents}
-            handleDeleteDocument={handleDeleteClick}
-          />
-        );
+        return <Documents handleDeleteDocument={handleDeleteClick} />;
      case 'Widgets':
        return (
          <Widgets
--- a/frontend/src/upload/Upload.tsx
+++ b/frontend/src/upload/Upload.tsx
@@ -166,7 +166,10 @@ function Upload({
                    dispatch(setSourceDocs(data));
                    dispatch(
                      setSelectedDocs(
-                        data?.find((d) => d.type?.toLowerCase() === 'local'),
+                        Array.isArray(data) &&
+                          data?.find(
+                            (d: Doc) => d.type?.toLowerCase() === 'local',
+                          ),
                      ),
                    );
                  });
@@ -182,15 +185,21 @@ function Upload({
                  getDocs().then((data) => {
                    dispatch(setSourceDocs(data));
                    const docIds = new Set(
-                      sourceDocs?.map((doc: Doc) => (doc.id ? doc.id : null)),
+                      (Array.isArray(sourceDocs) &&
+                        sourceDocs?.map((doc: Doc) =>
+                          doc.id ? doc.id : null,
+                        )) ||
+                        [],
                    );
-                    data?.map((updatedDoc: Doc) => {
-                      if (updatedDoc.id && !docIds.has(updatedDoc.id)) {
-                        //select the doc not present in the intersection of current Docs and fetched data
-                        dispatch(setSelectedDocs(updatedDoc));
-                        return;
-                      }
-                    });
+                    if (data && Array.isArray(data.docs)) {
+                      data.docs.map((updatedDoc: Doc) => {
+                        if (updatedDoc.id && !docIds.has(updatedDoc.id)) {
+                          // Select the doc not present in the intersection of current Docs and fetched data
+                          dispatch(setSelectedDocs(updatedDoc));
+                          return;
+                        }
+                      });
+                    }
                  });
                  setProgress(
                    (progress) =>