diff --git a/README.md b/README.md index eeecb598..99baf811 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ docker compose -f docker-compose-dev.yaml up -d > Make sure you have Python 3.10 or 3.11 installed. 1. Export required environment variables or prepare a `.env` file in the project folder: - - Copy [.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) and create `.env`. + - Copy [.env-template](https://github.com/arc53/DocsGPT/blob/main/application/.env-template) and create `.env`. (check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.) diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 182cdf2b..f109db26 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -241,6 +241,7 @@ def complete_stream( yield f"data: {data}\n\n" except Exception as e: print("\033[91merr", str(e), file=sys.stderr) + traceback.print_exc() data = json.dumps( { "type": "error", diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 8e62683e..6a2f3bea 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -2,11 +2,12 @@ import datetime import os import shutil import uuid +import math from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef from bson.objectid import ObjectId -from flask import Blueprint, jsonify, make_response, request +from flask import Blueprint, jsonify, make_response, request, redirect from flask_restx import inputs, fields, Namespace, Resource from werkzeug.utils import secure_filename @@ -315,7 +316,7 @@ class UploadFile(Resource): for file in files: filename = secure_filename(file.filename) file.save(os.path.join(temp_dir, filename)) - + print(f"Saved file: {filename}") zip_path = shutil.make_archive( base_name=os.path.join(save_dir, job_name), format="zip", @@ -323,6 +324,26 @@ class UploadFile(Resource): ) final_filename = os.path.basename(zip_path) shutil.rmtree(temp_dir) + task = ingest.delay( + settings.UPLOAD_FOLDER, + [ + ".rst", + ".md", + ".pdf", + ".txt", + ".docx", + ".csv", + ".epub", + ".html", + ".mdx", + ".json", + ".xlsx", + ".pptx", + ], + job_name, + final_filename, + user, + ) else: file = files[0] final_filename = secure_filename(file.filename) @@ -349,9 +370,10 @@ class UploadFile(Resource): final_filename, user, ) - except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + except Exception as err: + print(f"Error: {err}") + return make_response(jsonify({"success": False, "error": str(err)}), 400) return make_response(jsonify({"success": True, "task_id": task.id}), 200) @@ -422,6 +444,11 @@ class TaskStatus(Resource): task = celery.AsyncResult(task_id) task_meta = task.info + print(f"Task status: {task.status}") + if not isinstance( + task_meta, (dict, list, str, int, float, bool, type(None)) + ): + task_meta = str(task_meta) # Convert to a string representation except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) @@ -429,12 +456,70 @@ class TaskStatus(Resource): @user_ns.route("/api/combine") +class RedirectToSources(Resource): + @api.doc( + description="Redirects /api/combine to /api/sources for backward compatibility" + ) + def get(self): + return redirect("/api/sources", code=301) + + +@user_ns.route("/api/sources/paginated") +class PaginatedSources(Resource): + @api.doc(description="Get document with pagination, sorting and filtering") + def get(self): + user = "local" + sort_field = request.args.get("sort", "date") # Default to 'date' + sort_order = request.args.get("order", "desc") # Default to 'desc' + page = int(request.args.get("page", 1)) # Default to 1 + rows_per_page = int(request.args.get("rows", 10)) # Default to 10 + + # Prepare + query = {"user": user} + total_documents = sources_collection.count_documents(query) + total_pages = max(1, math.ceil(total_documents / rows_per_page)) + sort_order = 1 if sort_order == "asc" else -1 + skip = (page - 1) * rows_per_page + + try: + documents = ( + sources_collection.find(query) + .sort(sort_field, sort_order) + .skip(skip) + .limit(rows_per_page) + ) + + paginated_docs = [] + for doc in documents: + doc_data = { + "id": str(doc["_id"]), + "name": doc.get("name", ""), + "date": doc.get("date", ""), + "model": settings.EMBEDDINGS_NAME, + "location": "local", + "tokens": doc.get("tokens", ""), + "retriever": doc.get("retriever", "classic"), + "syncFrequency": doc.get("sync_frequency", ""), + } + paginated_docs.append(doc_data) + + response = { + "total": total_documents, + "totalPages": total_pages, + "currentPage": page, + "paginated": paginated_docs, + } + return make_response(jsonify(response), 200) + + except Exception as err: + return make_response(jsonify({"success": False, "error": str(err)}), 400) + + +@user_ns.route("/api/sources") class CombinedJson(Resource): @api.doc(description="Provide JSON file with combined available indexes") def get(self): user = "local" - sort_field = request.args.get('sort', 'date') # Default to 'date' - sort_order = request.args.get('order', "desc") # Default to 'desc' data = [ { "name": "default", @@ -447,7 +532,7 @@ class CombinedJson(Resource): ] try: - for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1): + for index in sources_collection.find({"user": user}).sort("date", -1): data.append( { "id": str(index["_id"]), @@ -485,6 +570,7 @@ class CombinedJson(Resource): "retriever": "brave_search", } ) + except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) @@ -1674,7 +1760,9 @@ class TextToSpeech(Resource): tts_model = api.model( "TextToSpeechModel", { - "text": fields.String(required=True, description="Text to be synthesized as audio"), + "text": fields.String( + required=True, description="Text to be synthesized as audio" + ), }, ) @@ -1686,8 +1774,15 @@ class TextToSpeech(Resource): try: tts_instance = GoogleTTS() audio_base64, detected_language = tts_instance.text_to_speech(text) - return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200) + return make_response( + jsonify( + { + "success": True, + "audio_base64": audio_base64, + "lang": detected_language, + } + ), + 200, + ) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - - diff --git a/application/llm/google_ai.py b/application/llm/google_ai.py new file mode 100644 index 00000000..df252abf --- /dev/null +++ b/application/llm/google_ai.py @@ -0,0 +1,48 @@ +from application.llm.base import BaseLLM + +class GoogleLLM(BaseLLM): + + def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): + + super().__init__(*args, **kwargs) + self.api_key = api_key + self.user_api_key = user_api_key + + def _clean_messages_google(self, messages): + return [ + { + "role": "model" if message["role"] == "system" else message["role"], + "parts": [message["content"]], + } + for message in messages[1:] + ] + + def _raw_gen( + self, + baseself, + model, + messages, + stream=False, + **kwargs + ): + import google.generativeai as genai + genai.configure(api_key=self.api_key) + model = genai.GenerativeModel(model, system_instruction=messages[0]["content"]) + response = model.generate_content(self._clean_messages_google(messages)) + return response.text + + def _raw_gen_stream( + self, + baseself, + model, + messages, + stream=True, + **kwargs + ): + import google.generativeai as genai + genai.configure(api_key=self.api_key) + model = genai.GenerativeModel(model, system_instruction=messages[0]["content"]) + response = model.generate_content(self._clean_messages_google(messages), stream=True) + for line in response: + if line.text is not None: + yield line.text \ No newline at end of file diff --git a/application/llm/llm_creator.py b/application/llm/llm_creator.py index 6a19de10..f32089de 100644 --- a/application/llm/llm_creator.py +++ b/application/llm/llm_creator.py @@ -6,6 +6,7 @@ from application.llm.llama_cpp import LlamaCpp from application.llm.anthropic import AnthropicLLM from application.llm.docsgpt_provider import DocsGPTAPILLM from application.llm.premai import PremAILLM +from application.llm.google_ai import GoogleLLM class LLMCreator: @@ -18,7 +19,8 @@ class LLMCreator: "anthropic": AnthropicLLM, "docsgpt": DocsGPTAPILLM, "premai": PremAILLM, - "groq": GroqLLM + "groq": GroqLLM, + "google": GoogleLLM } @classmethod diff --git a/application/parser/remote/reddit_loader.py b/application/parser/remote/reddit_loader.py index 0230653a..22f5b185 100644 --- a/application/parser/remote/reddit_loader.py +++ b/application/parser/remote/reddit_loader.py @@ -1,10 +1,19 @@ from application.parser.remote.base import BaseRemote from langchain_community.document_loaders import RedditPostsLoader +import json class RedditPostsLoaderRemote(BaseRemote): def load_data(self, inputs): - data = eval(inputs) + try: + data = json.loads(inputs) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON input: {e}") + + required_fields = ["client_id", "client_secret", "user_agent", "search_queries"] + missing_fields = [field for field in required_fields if field not in data] + if missing_fields: + raise ValueError(f"Missing required fields: {', '.join(missing_fields)}") client_id = data.get("client_id") client_secret = data.get("client_secret") user_agent = data.get("user_agent") diff --git a/application/retriever/classic_rag.py b/application/retriever/classic_rag.py index 6a67cb38..42e318d2 100644 --- a/application/retriever/classic_rag.py +++ b/application/retriever/classic_rag.py @@ -45,7 +45,6 @@ class ClassicRAG(BaseRetriever): settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY ) docs_temp = docsearch.search(self.question, k=self.chunks) - print(docs_temp) docs = [ { "title": i.metadata.get( @@ -60,8 +59,6 @@ class ClassicRAG(BaseRetriever): } for i in docs_temp ] - if settings.LLM_NAME == "llama.cpp": - docs = [docs[0]] return docs diff --git a/docs/pages/_app.mdx b/docs/pages/_app.mdx index ac2be195..1cb8cadd 100644 --- a/docs/pages/_app.mdx +++ b/docs/pages/_app.mdx @@ -4,7 +4,7 @@ export default function MyApp({ Component, pageProps }) { return ( <> - + > ) } \ No newline at end of file diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index 3591469b..242efb1a 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -18,6 +18,7 @@ import SourceDropdown from './components/SourceDropdown'; import { setConversation, updateConversationId, + handleAbort, } from './conversation/conversationSlice'; import ConversationTile from './conversation/ConversationTile'; import { useDarkTheme, useMediaQuery, useOutsideAlerter } from './hooks'; @@ -34,10 +35,12 @@ import { selectSelectedDocs, selectSelectedDocsStatus, selectSourceDocs, + selectPaginatedDocuments, setConversations, setModalStateDeleteConv, setSelectedDocs, setSourceDocs, + setPaginatedDocuments, } from './preferences/preferenceSlice'; import Spinner from './assets/spinner.svg'; import SpinnerDark from './assets/spinner-dark.svg'; @@ -72,6 +75,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { const conversations = useSelector(selectConversations); const modalStateDeleteConv = useSelector(selectModalStateDeleteConv); const conversationId = useSelector(selectConversationId); + const paginatedDocuments = useSelector(selectPaginatedDocuments); const [isDeletingConversation, setIsDeletingConversation] = useState(false); const { isMobile } = useMediaQuery(); @@ -143,9 +147,18 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { }) .then((updatedDocs) => { dispatch(setSourceDocs(updatedDocs)); + const updatedPaginatedDocs = paginatedDocuments?.filter( + (document) => document.id !== doc.id, + ); + dispatch( + setPaginatedDocuments(updatedPaginatedDocs || paginatedDocuments), + ); dispatch( setSelectedDocs( - updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'), + Array.isArray(updatedDocs) && + updatedDocs?.find( + (doc: Doc) => doc.name.toLowerCase() === 'default', + ), ), ); }) @@ -168,6 +181,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { }; const resetConversation = () => { + handleAbort(); dispatch(setConversation([])); dispatch( updateConversationId({ diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index 84674049..4e7112d0 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -1,7 +1,8 @@ const endpoints = { USER: { - DOCS: '/api/combine', + DOCS: '/api/sources', DOCS_CHECK: '/api/docs_check', + DOCS_PAGINATED: '/api/sources/paginated', API_KEYS: '/api/get_api_keys', CREATE_API_KEY: '/api/create_api_key', DELETE_API_KEY: '/api/delete_api_key', diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index 53b38f50..942318ae 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -2,8 +2,9 @@ import apiClient from '../client'; import endpoints from '../endpoints'; const userService = { - getDocs: (sort = 'date', order = 'desc'): Promise => - apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`), + getDocs: (): Promise => apiClient.get(`${endpoints.USER.DOCS}`), + getDocsWithPagination: (query: string): Promise => + apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`), checkDocs: (data: any): Promise => apiClient.post(endpoints.USER.DOCS_CHECK, data), getAPIKeys: (): Promise => apiClient.get(endpoints.USER.API_KEYS), diff --git a/frontend/src/assets/double-arrow-left.svg b/frontend/src/assets/double-arrow-left.svg new file mode 100644 index 00000000..cab9ff90 --- /dev/null +++ b/frontend/src/assets/double-arrow-left.svg @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/double-arrow-right.svg b/frontend/src/assets/double-arrow-right.svg new file mode 100644 index 00000000..0d5167c2 --- /dev/null +++ b/frontend/src/assets/double-arrow-right.svg @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/single-left-arrow.svg b/frontend/src/assets/single-left-arrow.svg new file mode 100644 index 00000000..f28b2592 --- /dev/null +++ b/frontend/src/assets/single-left-arrow.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/src/assets/single-right-arrow.svg b/frontend/src/assets/single-right-arrow.svg new file mode 100644 index 00000000..85729e57 --- /dev/null +++ b/frontend/src/assets/single-right-arrow.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/DocumentPagination.tsx b/frontend/src/components/DocumentPagination.tsx new file mode 100644 index 00000000..b0532362 --- /dev/null +++ b/frontend/src/components/DocumentPagination.tsx @@ -0,0 +1,119 @@ +import React, { useState } from 'react'; +import SingleArrowLeft from '../assets/single-left-arrow.svg'; +import SingleArrowRight from '../assets/single-right-arrow.svg'; +import DoubleArrowLeft from '../assets/double-arrow-left.svg'; +import DoubleArrowRight from '../assets/double-arrow-right.svg'; + +interface PaginationProps { + currentPage: number; + totalPages: number; + rowsPerPage: number; + onPageChange: (page: number) => void; + onRowsPerPageChange: (rows: number) => void; +} + +const Pagination: React.FC = ({ + currentPage, + totalPages, + rowsPerPage, + onPageChange, + onRowsPerPageChange, +}) => { + const [rowsPerPageOptions] = useState([5, 10, 15, 20]); + + const handlePreviousPage = () => { + if (currentPage > 1) { + onPageChange(currentPage - 1); + } + }; + + const handleNextPage = () => { + if (currentPage < totalPages) { + onPageChange(currentPage + 1); + } + }; + + const handleFirstPage = () => { + onPageChange(1); + }; + + const handleLastPage = () => { + onPageChange(totalPages); + }; + + return ( + + + Rows per page: + onRowsPerPageChange(Number(e.target.value))} + className="border border-gray-300 rounded px-2 py-1 dark:bg-dark-charcoal dark:text-gray-50" + > + {rowsPerPageOptions.map((option) => ( + + {option} + + ))} + + + + + Page {currentPage} of {totalPages} + + + + + + + + + + + + + + + + + + ); +}; + +export default Pagination; diff --git a/frontend/src/components/Dropdown.tsx b/frontend/src/components/Dropdown.tsx index 07f33650..15923661 100644 --- a/frontend/src/components/Dropdown.tsx +++ b/frontend/src/components/Dropdown.tsx @@ -140,7 +140,7 @@ function Dropdown({ : option.description }`} - {showEdit && onEdit && ( + {showEdit && onEdit && option.type !== 'public' && ( - + { if (e.key === 'Enter' && !e.shiftKey) { diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx index 284c2b56..8e5df666 100644 --- a/frontend/src/conversation/ConversationBubble.tsx +++ b/frontend/src/conversation/ConversationBubble.tsx @@ -1,12 +1,14 @@ +import 'katex/dist/katex.min.css'; + import { forwardRef, useState } from 'react'; import ReactMarkdown from 'react-markdown'; import { useSelector } from 'react-redux'; import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'; import { vscDarkPlus } from 'react-syntax-highlighter/dist/cjs/styles/prism'; +import rehypeKatex from 'rehype-katex'; import remarkGfm from 'remark-gfm'; import remarkMath from 'remark-math'; -import rehypeKatex from 'rehype-katex'; -import 'katex/dist/katex.min.css'; + import DocsGPT3 from '../assets/cute_docsgpt3.svg'; import Dislike from '../assets/dislike.svg?react'; import Document from '../assets/document.svg'; @@ -16,13 +18,13 @@ import Sources from '../assets/sources.svg'; import Avatar from '../components/Avatar'; import CopyButton from '../components/CopyButton'; import Sidebar from '../components/Sidebar'; +import SpeakButton from '../components/TextToSpeechButton'; import { selectChunks, selectSelectedDocs, } from '../preferences/preferenceSlice'; import classes from './ConversationBubble.module.css'; import { FEEDBACK, MESSAGE_TYPE } from './conversationModels'; -import SpeakButton from '../components/TextToSpeechButton'; const DisableSourceFE = import.meta.env.VITE_DISABLE_SOURCE_FE || false; @@ -41,6 +43,7 @@ const ConversationBubble = forwardRef< { message, type, className, feedback, handleFeedback, sources, retryBtn }, ref, ) { + // const bubbleRef = useRef(null); const chunks = useSelector(selectChunks); const selectedDocs = useSelector(selectSelectedDocs); const [isLikeHovered, setIsLikeHovered] = useState(false); @@ -141,7 +144,7 @@ const ConversationBubble = forwardRef< /> Sources - + {sources?.slice(0, 3)?.map((source, index) => ( @@ -190,7 +193,7 @@ const ConversationBubble = forwardRef< {activeTooltip === index && ( setActiveTooltip(index)} onMouseOut={() => setActiveTooltip(null)} > @@ -231,14 +234,14 @@ const ConversationBubble = forwardRef< Answer ( 'fetchAnswer', - async ({ question }, { dispatch, getState, signal }) => { + async ({ question }, { dispatch, getState }) => { + if (abortController) { + abortController.abort(); + } + abortController = new AbortController(); + const { signal } = abortController; + let isSourceUpdated = false; const state = getState() as RootState; if (state.preference) { diff --git a/frontend/src/hooks/useDefaultDocument.ts b/frontend/src/hooks/useDefaultDocument.ts index 37374ce0..7f4b9812 100644 --- a/frontend/src/hooks/useDefaultDocument.ts +++ b/frontend/src/hooks/useDefaultDocument.ts @@ -17,11 +17,12 @@ export default function useDefaultDocument() { getDocs().then((data) => { dispatch(setSourceDocs(data)); if (!selectedDoc) - data?.forEach((doc: Doc) => { - if (doc.model && doc.name === 'default') { - dispatch(setSelectedDocs(doc)); - } - }); + Array.isArray(data) && + data?.forEach((doc: Doc) => { + if (doc.model && doc.name === 'default') { + dispatch(setSelectedDocs(doc)); + } + }); }); }; diff --git a/frontend/src/index.css b/frontend/src/index.css index 4319403e..9bd2ec96 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -50,11 +50,11 @@ body.dark { @layer components { .table-default { - @apply block w-max table-auto content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray; + @apply block w-full mx-auto table-auto content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray overflow-auto; } .table-default th { - @apply p-4 w-[244px] font-normal text-gray-400; /* Remove border-r */ + @apply p-4 w-full font-normal text-gray-400 text-nowrap; /* Remove border-r */ } .table-default th:last-child { @@ -514,3 +514,29 @@ input:-webkit-autofill:focus { .logs-table { font-family: 'IBMPlexMono-Medium', system-ui; } + +.fade-in { + animation: fadeIn 0.5s ease-in-out; +} + +@keyframes fadeIn { + 0% { + opacity: 0; + } + 100% { + opacity: 1; + } +} + +.fade-in-bubble { + opacity: 0; + transform: translateY(10px); + animation: fadeInUp 0.5s forwards; +} + +@keyframes fadeInUp { + to { + opacity: 1; + transform: translateY(0); + } +} diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts index 9affd0ab..0d9c8931 100644 --- a/frontend/src/models/misc.ts +++ b/frontend/src/models/misc.ts @@ -14,6 +14,13 @@ export type Doc = { syncFrequency?: string; }; +export type GetDocsResponse = { + docs: Doc[]; + totalDocuments: number; + totalPages: number; + nextCursor: string; +}; + export type PromptProps = { prompts: { name: string; id: string; type: string }[]; selectedPrompt: { name: string; id: string; type: string }; @@ -22,7 +29,7 @@ export type PromptProps = { }; export type DocumentsProps = { - documents: Doc[] | null; + paginatedDocuments: Doc[] | null; handleDeleteDocument: (index: number, document: Doc) => void; }; diff --git a/frontend/src/preferences/PromptsModal.tsx b/frontend/src/preferences/PromptsModal.tsx index c207e0be..3aa8c54c 100644 --- a/frontend/src/preferences/PromptsModal.tsx +++ b/frontend/src/preferences/PromptsModal.tsx @@ -1,6 +1,7 @@ import { ActiveState } from '../models/misc'; import Exit from '../assets/exit.svg'; import Input from '../components/Input'; +import React from 'react'; function AddPrompt({ setModalState, @@ -9,6 +10,7 @@ function AddPrompt({ setNewPromptName, newPromptContent, setNewPromptContent, + disableSave, }: { setModalState: (state: ActiveState) => void; handleAddPrompt?: () => void; @@ -16,6 +18,7 @@ function AddPrompt({ setNewPromptName: (name: string) => void; newPromptContent: string; setNewPromptContent: (content: string) => void; + disableSave: boolean; }) { return ( @@ -23,6 +26,8 @@ function AddPrompt({ className="absolute top-3 right-4 m-2 w-3" onClick={() => { setModalState('INACTIVE'); + setNewPromptName(''); + setNewPromptContent(''); }} > @@ -41,7 +46,7 @@ function AddPrompt({ className="h-10 rounded-lg" value={newPromptName} onChange={(e) => setNewPromptName(e.target.value)} - > + /> Prompt Name @@ -62,6 +67,8 @@ function AddPrompt({ Save @@ -79,6 +86,7 @@ function EditPrompt({ editPromptContent, setEditPromptContent, currentPromptEdit, + disableSave, }: { setModalState: (state: ActiveState) => void; handleEditPrompt?: (id: string, type: string) => void; @@ -87,6 +95,7 @@ function EditPrompt({ editPromptContent: string; setEditPromptContent: (content: string) => void; currentPromptEdit: { name: string; id: string; type: string }; + disableSave: boolean; }) { return ( @@ -140,7 +149,8 @@ function EditPrompt({ handleEditPrompt && handleEditPrompt(currentPromptEdit.id, currentPromptEdit.type); }} - disabled={currentPromptEdit.type === 'public'} + disabled={currentPromptEdit.type === 'public' || disableSave} + title={disableSave && editPromptName ? 'Name already exists' : ''} > Save @@ -151,6 +161,7 @@ function EditPrompt({ } export default function PromptsModal({ + existingPrompts, modalState, setModalState, type, @@ -166,6 +177,7 @@ export default function PromptsModal({ handleAddPrompt, handleEditPrompt, }: { + existingPrompts: { name: string; id: string; type: string }[]; modalState: ActiveState; setModalState: (state: ActiveState) => void; type: 'ADD' | 'EDIT'; @@ -181,6 +193,25 @@ export default function PromptsModal({ handleAddPrompt?: () => void; handleEditPrompt?: (id: string, type: string) => void; }) { + const [disableSave, setDisableSave] = React.useState(true); + const handlePrompNameChange = (edit: boolean, newName: string) => { + const nameExists = existingPrompts.find( + (prompt) => newName === prompt.name, + ); + + if (newName && !nameExists) { + setDisableSave(false); + } else { + setDisableSave(true); + } + + if (edit) { + setEditPromptName(newName); + } else { + setNewPromptName(newName); + } + }; + let view; if (type === 'ADD') { @@ -189,9 +220,10 @@ export default function PromptsModal({ setModalState={setModalState} handleAddPrompt={handleAddPrompt} newPromptName={newPromptName} - setNewPromptName={setNewPromptName} + setNewPromptName={handlePrompNameChange.bind(null, false)} newPromptContent={newPromptContent} setNewPromptContent={setNewPromptContent} + disableSave={disableSave} /> ); } else if (type === 'EDIT') { @@ -200,10 +232,11 @@ export default function PromptsModal({ setModalState={setModalState} handleEditPrompt={handleEditPrompt} editPromptName={editPromptName} - setEditPromptName={setEditPromptName} + setEditPromptName={handlePrompNameChange.bind(null, true)} editPromptContent={editPromptContent} setEditPromptContent={setEditPromptContent} currentPromptEdit={currentPromptEdit} + disableSave={disableSave} /> ); } else { diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index af1060b8..32cf8b17 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -1,18 +1,14 @@ import conversationService from '../api/services/conversationService'; import userService from '../api/services/userService'; -import { Doc } from '../models/misc'; +import { Doc, GetDocsResponse } from '../models/misc'; //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later. -export async function getDocs( - sort = 'date', - order = 'desc', -): Promise { +export async function getDocs(): Promise { try { - const response = await userService.getDocs(sort, order); + const response = await userService.getDocs(); const data = await response.json(); const docs: Doc[] = []; - data.forEach((doc: object) => { docs.push(doc as Doc); }); @@ -24,6 +20,33 @@ export async function getDocs( } } +export async function getDocsWithPagination( + sort = 'date', + order = 'desc', + pageNumber = 1, + rowsPerPage = 10, +): Promise { + try { + const query = `sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`; + const response = await userService.getDocsWithPagination(query); + const data = await response.json(); + const docs: Doc[] = []; + Array.isArray(data.paginated) && + data.paginated.forEach((doc: Doc) => { + docs.push(doc as Doc); + }); + return { + docs: docs, + totalDocuments: data.total, + totalPages: data.totalPages, + nextCursor: data.nextCursor, + }; + } catch (error) { + console.log(error); + return null; + } +} + export async function getConversations(): Promise<{ data: { name: string; id: string }[] | null; loading: boolean; diff --git a/frontend/src/preferences/preferenceSlice.ts b/frontend/src/preferences/preferenceSlice.ts index c566ba70..8b3064d5 100644 --- a/frontend/src/preferences/preferenceSlice.ts +++ b/frontend/src/preferences/preferenceSlice.ts @@ -20,6 +20,7 @@ export interface Preference { loading: boolean; }; modalState: ActiveState; + paginatedDocuments: Doc[] | null; } const initialState: Preference = { @@ -42,6 +43,7 @@ const initialState: Preference = { loading: false, }, modalState: 'INACTIVE', + paginatedDocuments: null, }; export const prefSlice = createSlice({ @@ -57,6 +59,9 @@ export const prefSlice = createSlice({ setSourceDocs: (state, action) => { state.sourceDocs = action.payload; }, + setPaginatedDocuments: (state, action) => { + state.paginatedDocuments = action.payload; + }, setConversations: (state, action) => { state.conversations = action.payload; }, @@ -84,6 +89,7 @@ export const { setChunks, setTokenLimit, setModalStateDeleteConv, + setPaginatedDocuments, } = prefSlice.actions; export default prefSlice.reducer; @@ -155,3 +161,5 @@ export const selectPrompt = (state: RootState) => state.preference.prompt; export const selectChunks = (state: RootState) => state.preference.chunks; export const selectTokenLimit = (state: RootState) => state.preference.token_limit; +export const selectPaginatedDocuments = (state: RootState) => + state.preference.paginatedDocuments; diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index f94d1a87..f91a3355 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -1,19 +1,20 @@ -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; import PropTypes from 'prop-types'; -import { useTranslation } from 'react-i18next'; -import { useDispatch } from 'react-redux'; - import userService from '../api/services/userService'; import SyncIcon from '../assets/sync.svg'; import Trash from '../assets/trash.svg'; import caretSort from '../assets/caret-sort.svg'; import DropdownMenu from '../components/DropdownMenu'; -import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported import SkeletonLoader from '../components/SkeletonLoader'; -import { getDocs } from '../preferences/preferenceApi'; -import { setSourceDocs } from '../preferences/preferenceSlice'; import Input from '../components/Input'; import Upload from '../upload/Upload'; // Import the Upload component +import Pagination from '../components/DocumentPagination'; +import { useTranslation } from 'react-i18next'; +import { useDispatch } from 'react-redux'; +import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported +import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi'; +import { setSourceDocs } from '../preferences/preferenceSlice'; +import { setPaginatedDocuments } from '../preferences/preferenceSlice'; // Utility function to format numbers const formatTokens = (tokens: number): string => { @@ -33,12 +34,11 @@ const formatTokens = (tokens: number): string => { }; const Documents: React.FC = ({ - documents, + paginatedDocuments, handleDeleteDocument, }) => { const { t } = useTranslation(); const dispatch = useDispatch(); - // State for search input const [searchTerm, setSearchTerm] = useState(''); // State for modal: active/inactive @@ -47,37 +47,49 @@ const Documents: React.FC = ({ const [loading, setLoading] = useState(false); const [sortField, setSortField] = useState<'date' | 'tokens'>('date'); const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('desc'); + // Pagination + const [currentPage, setCurrentPage] = useState(1); + const [rowsPerPage, setRowsPerPage] = useState(10); + const [totalPages, setTotalPages] = useState(1); + // const [totalDocuments, setTotalDocuments] = useState(0); + // Filter documents based on the search term + const filteredDocuments = paginatedDocuments?.filter((document) => + document.name.toLowerCase().includes(searchTerm.toLowerCase()), + ); + // State for documents + const currentDocuments = filteredDocuments ?? []; + console.log('currentDocuments', currentDocuments); const syncOptions = [ { label: 'Never', value: 'never' }, { label: 'Daily', value: 'daily' }, { label: 'Weekly', value: 'weekly' }, { label: 'Monthly', value: 'monthly' }, ]; - const refreshDocs = (field: 'date' | 'tokens') => { - if (field === sortField) { - setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc'); - } else { - setSortOrder('desc'); - setSortField(field); + + const refreshDocs = ( + field: 'date' | 'tokens' | undefined, + pageNumber?: number, + rows?: number, + ) => { + const page = pageNumber ?? currentPage; + const rowsPerPg = rows ?? rowsPerPage; + + if (field !== undefined) { + if (field === sortField) { + // Toggle sort order + setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc'); + } else { + // Change sort field and reset order to 'desc' + setSortField(field); + setSortOrder('desc'); + } } - getDocs(sortField, sortOrder) + getDocsWithPagination(sortField, sortOrder, page, rowsPerPg) .then((data) => { - dispatch(setSourceDocs(data)); - }) - .catch((error) => console.error(error)) - .finally(() => { - setLoading(false); - }); - }; - const handleManageSync = (doc: Doc, sync_frequency: string) => { - setLoading(true); - userService - .manageSync({ source_id: doc.id, sync_frequency }) - .then(() => { - return getDocs(); - }) - .then((data) => { - dispatch(setSourceDocs(data)); + //dispatch(setSourceDocs(data ? data.docs : [])); + dispatch(setPaginatedDocuments(data ? data.docs : [])); + setTotalPages(data ? data.totalPages : 0); + //setTotalDocuments(data ? data.totalDocuments : 0); }) .catch((error) => console.error(error)) .finally(() => { @@ -85,10 +97,40 @@ const Documents: React.FC = ({ }); }; - // Filter documents based on the search term - const filteredDocuments = documents?.filter((document) => - document.name.toLowerCase().includes(searchTerm.toLowerCase()), - ); + const handleManageSync = (doc: Doc, sync_frequency: string) => { + setLoading(true); + userService + .manageSync({ source_id: doc.id, sync_frequency }) + .then(() => { + // First, fetch the updated source docs + return getDocs(); + }) + .then((data) => { + dispatch(setSourceDocs(data)); + return getDocsWithPagination( + sortField, + sortOrder, + currentPage, + rowsPerPage, + ); + }) + .then((paginatedData) => { + dispatch( + setPaginatedDocuments(paginatedData ? paginatedData.docs : []), + ); + setTotalPages(paginatedData ? paginatedData.totalPages : 0); + }) + .catch((error) => console.error('Error in handleManageSync:', error)) + .finally(() => { + setLoading(false); + }); + }; + + useEffect(() => { + if (modalState === 'INACTIVE') { + refreshDocs(sortField, currentPage, rowsPerPage); + } + }, [modalState, sortField, currentPage, rowsPerPage]); return ( @@ -154,16 +196,16 @@ const Documents: React.FC = ({ - {!filteredDocuments?.length && ( + {!currentDocuments?.length && ( {t('settings.documents.noData')} )} - {filteredDocuments && - filteredDocuments.map((document, index) => ( - + {Array.isArray(currentDocuments) && + currentDocuments.map((document, index) => ( + {document.name} {document.date} @@ -173,7 +215,7 @@ const Documents: React.FC = ({ {document.type === 'remote' ? 'Pre-loaded' : 'Private'} - + {document.type !== 'remote' && ( = ({ )} + {/* Pagination component with props: + # Note: Every time the page changes, + the refreshDocs function is called with the updated page number and rows per page. + and reset cursor paginated query parameter to undefined. + */} + { + setCurrentPage(page); + refreshDocs(sortField, page, rowsPerPage); + }} + onRowsPerPageChange={(rows) => { + setRowsPerPage(rows); + setCurrentPage(1); + refreshDocs(sortField, 1, rows); + }} + /> ); }; Documents.propTypes = { - documents: PropTypes.array.isRequired, + //documents: PropTypes.array.isRequired, handleDeleteDocument: PropTypes.func.isRequired, }; diff --git a/frontend/src/settings/Prompts.tsx b/frontend/src/settings/Prompts.tsx index 3d041555..6e1810e5 100644 --- a/frontend/src/settings/Prompts.tsx +++ b/frontend/src/settings/Prompts.tsx @@ -58,7 +58,8 @@ export default function Prompts({ } setModalState('INACTIVE'); onSelectPrompt(newPromptName, newPrompt.id, newPromptContent); - setNewPromptName(newPromptName); + setNewPromptName(''); + setNewPromptContent(''); } catch (error) { console.error(error); } @@ -178,6 +179,7 @@ export default function Prompts({ { setWidgetScreenshot(screenshot); }; + const updateDocumentsList = (documents: Doc[], index: number) => [ + ...documents.slice(0, index), + ...documents.slice(index + 1), + ]; + const handleDeleteClick = (index: number, doc: Doc) => { userService .deletePath(doc.id ?? '') .then((response) => { if (response.ok && documents) { - const updatedDocuments = [ - ...documents.slice(0, index), - ...documents.slice(index + 1), - ]; - dispatch(setSourceDocs(updatedDocuments)); + if (paginatedDocuments) { + dispatch( + setPaginatedDocuments( + updateDocumentsList(paginatedDocuments, index), + ), + ); + } + dispatch(setSourceDocs(updateDocumentsList(documents, index))); } }) .catch((error) => console.error(error)); @@ -72,7 +83,7 @@ export default function Settings() { case t('settings.documents.label'): return ( ); diff --git a/frontend/src/store.ts b/frontend/src/store.ts index 5843d493..8f426ed6 100644 --- a/frontend/src/store.ts +++ b/frontend/src/store.ts @@ -38,6 +38,7 @@ const preloadedState: { preference: Preference } = { }, ], modalState: 'INACTIVE', + paginatedDocuments: null, }, }; const store = configureStore({ diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 01ecc480..2f28042a 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -76,7 +76,7 @@ function Upload({ d.type?.toLowerCase() === 'local'), + Array.isArray(data) && + data?.find( + (d: Doc) => d.type?.toLowerCase() === 'local', + ), ), ); }); @@ -182,15 +185,21 @@ function Upload({ getDocs().then((data) => { dispatch(setSourceDocs(data)); const docIds = new Set( - sourceDocs?.map((doc: Doc) => (doc.id ? doc.id : null)), + (Array.isArray(sourceDocs) && + sourceDocs?.map((doc: Doc) => + doc.id ? doc.id : null, + )) || + [], ); - data?.map((updatedDoc: Doc) => { - if (updatedDoc.id && !docIds.has(updatedDoc.id)) { - //select the doc not present in the intersection of current Docs and fetched data - dispatch(setSelectedDocs(updatedDoc)); - return; - } - }); + if (data && Array.isArray(data)) { + data.map((updatedDoc: Doc) => { + if (updatedDoc.id && !docIds.has(updatedDoc.id)) { + // Select the doc not present in the intersection of current Docs and fetched data + dispatch(setSelectedDocs(updatedDoc)); + return; + } + }); + } }); setProgress( (progress) =>
Sources
Answer