LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 16:43:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Paginated With MongoDB / Create New Endpoint

Browse Source 
change routes /combine name, add route /api/source/paginated
add new endpoint source/paginated
fixing table responsive
create new function to handling api/source/paginated
This commit is contained in:
fadingNA
2024-11-08 11:02:13 -05:00
parent 4429755c09
commit 5debb48265
11 changed files with 173 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
frontend/src/Navigation.tsx
View File

@@ -145,7 +145,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
dispatch(setSourceDocs(updatedDocs));
dispatch(
setSelectedDocs(
updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'),
Array.isArray(updatedDocs) &&
updatedDocs?.find(
(doc: Doc) => doc.name.toLowerCase() === 'default',
),
),
);
})

3
frontend/src/api/endpoints.ts
View File

@@ -1,7 +1,8 @@
const endpoints = {
USER: {
DOCS: '/api/combine',
DOCS: '/api/sources',
DOCS_CHECK: '/api/docs_check',
DOCS_PAGINATED: '/api/sources/paginated',
API_KEYS: '/api/get_api_keys',
CREATE_API_KEY: '/api/create_api_key',
DELETE_API_KEY: '/api/delete_api_key',

13
frontend/src/api/services/userService.ts
View File

@@ -2,15 +2,10 @@ import apiClient from '../client';
import endpoints from '../endpoints';
const userService = {
getDocs: (
sort: string,
order: string,
pageNumber: number,
rowsPerPage: number,
): Promise<any> =>
apiClient.get(
`${endpoints.USER.DOCS}?sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`,
),
getDocs: (sort: string, order: string): Promise<any> =>
apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`),
getDocsWithPagination: (query: string): Promise<any> =>
apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`),
checkDocs: (data: any): Promise<any> =>
apiClient.post(endpoints.USER.DOCS_CHECK, data),
getAPIKeys: (): Promise<any> => apiClient.get(endpoints.USER.API_KEYS),

11
frontend/src/hooks/useDefaultDocument.ts
View File

@@ -17,11 +17,12 @@ export default function useDefaultDocument() {
getDocs().then((data) => {
dispatch(setSourceDocs(data));
if (!selectedDoc)
data?.forEach((doc: Doc) => {
if (doc.model && doc.name === 'default') {
dispatch(setSelectedDocs(doc));
}
});
Array.isArray(data) &&
data?.forEach((doc: Doc) => {
if (doc.model && doc.name === 'default') {
dispatch(setSelectedDocs(doc));
}
});
});
};

4
frontend/src/index.css
View File

@@ -50,11 +50,11 @@ body.dark {
@layer components {
.table-default {
@apply block w-max mx-auto table-fixed content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
@apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
}
.table-default th {
@apply p-4 w-[250px] font-normal text-gray-400; /* Remove border-r */
@apply p-4 w-1/4 font-normal text-gray-400 text-nowrap; /* Remove border-r */
}
.table-default th:last-child {

2
frontend/src/models/misc.ts
View File

@@ -18,6 +18,7 @@ export type GetDocsResponse = {
docs: Doc[];
totalDocuments: number;
totalPages: number;
nextCursor: string;
};
export type PromptProps = {
@@ -28,7 +29,6 @@ export type PromptProps = {
};
export type DocumentsProps = {
documents: Doc[] | null;
handleDeleteDocument: (index: number, document: Doc) => void;
};

76
frontend/src/preferences/preferenceApi.ts
View File

@@ -4,49 +4,53 @@ import { Doc, GetDocsResponse } from '../models/misc';
//Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
export async function getDocs(
sort?: string,
order?: string,
pageNumber?: number,
rowsPerPage?: number,
withPagination?: true,
): Promise<GetDocsResponse | null>;
sort = 'date',
order = 'desc',
): Promise<Doc[] | null> {
try {
const response = await userService.getDocs(sort, order);
const data = await response.json();
export async function getDocs(
const docs: Doc[] = [];
console.log(data);
data.forEach((doc: object) => {
docs.push(doc as Doc);
});
return docs;
} catch (error) {
console.log(error);
return null;
}
}
export async function getDocsWithPagination(
sort = 'date',
order = 'desc',
pageNumber = 1,
rowsPerPage = 5,
withPagination = false,
): Promise<Doc[] | GetDocsResponse | null> {
rowsPerPage = 10,
): Promise<GetDocsResponse | null> {
try {
const response = await userService.getDocs(
sort,
order,
pageNumber,
rowsPerPage,
);
const query = `sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`;
const response = await userService.getDocsWithPagination(query);
const data = await response.json();
console.log(data);
if (withPagination) {
const docs: Doc[] = [];
Array.isArray(data.paginated_docs) &&
data.paginated_docs.forEach((doc: object) => {
docs.push(doc as Doc);
});
const totalDocuments = data.totalDocuments || 0;
const totalPages = data.totalPages || 0;
console.log(`totalDocuments: ${totalDocuments}`);
console.log(`totalPages: ${totalPages}`);
return { docs, totalDocuments, totalPages };
} else {
const docs: Doc[] = [];
Array.isArray(data.documents) &&
data.documents.forEach((doc: object) => {
docs.push(doc as Doc);
});
return docs;
}
const docs: Doc[] = [];
console.log(`data: ${data}`);
Array.isArray(data.paginated) &&
data.paginated.forEach((doc: Doc) => {
docs.push(doc as Doc);
});
console.log(`total: ${data.total}`);
console.log(`totalPages: ${data.totalPages}`);
console.log(`cursor: ${data.nextCursor}`);
console.log(`currentPage: ${data.currentPage}`);
return {
docs: docs,
totalDocuments: data.total,
totalPages: data.totalPages,
nextCursor: data.nextCursor,
};
} catch (error) {
console.log(error);
return null;

40
frontend/src/settings/Documents.tsx
View File

@@ -10,7 +10,7 @@ import caretSort from '../assets/caret-sort.svg';
import DropdownMenu from '../components/DropdownMenu';
import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported
import SkeletonLoader from '../components/SkeletonLoader';
import { getDocs } from '../preferences/preferenceApi';
import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi';
import { setSourceDocs } from '../preferences/preferenceSlice';
import Input from '../components/Input';
import Upload from '../upload/Upload'; // Import the Upload component
@@ -33,13 +33,9 @@ const formatTokens = (tokens: number): string => {
}
};
const Documents: React.FC<DocumentsProps> = ({
documents,
handleDeleteDocument,
}) => {
const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
const { t } = useTranslation();
const dispatch = useDispatch();
// State for search input
const [searchTerm, setSearchTerm] = useState('');
// State for modal: active/inactive
@@ -60,7 +56,6 @@ const Documents: React.FC<DocumentsProps> = ({
);
// State for documents
const currentDocuments = filteredDocuments ?? [];
const syncOptions = [
{ label: 'Never', value: 'never' },
{ label: 'Daily', value: 'daily' },
@@ -73,9 +68,9 @@ const Documents: React.FC<DocumentsProps> = ({
pageNumber?: number,
rows?: number,
) => {
console.log(`field: ${field}, pageNumber: ${pageNumber}, rows: ${rows}`);
const page = pageNumber ?? currentPage;
const rowsPerPg = rows ?? rowsPerPage;
if (field !== undefined) {
if (field === sortField) {
// Toggle sort order
@@ -86,9 +81,9 @@ const Documents: React.FC<DocumentsProps> = ({
setSortOrder('desc');
}
}
getDocs(sortField, sortOrder, page, rowsPerPg, true)
getDocsWithPagination(sortField, sortOrder, page, rowsPerPg)
.then((data) => {
console.log(data);
console.log('Data received from getDocsWithPagination:', data);
dispatch(setSourceDocs(data ? data.docs : []));
setFetchedDocuments(data ? data.docs : []);
setTotalPages(data ? data.totalPages : 0);
@@ -99,6 +94,7 @@ const Documents: React.FC<DocumentsProps> = ({
setLoading(false);
});
};
const handleManageSync = (doc: Doc, sync_frequency: string) => {
setLoading(true);
userService
@@ -114,9 +110,13 @@ const Documents: React.FC<DocumentsProps> = ({
setLoading(false);
});
};
useEffect(() => {
refreshDocs(sortField, currentPage, rowsPerPage);
}, []);
if (modalState === 'INACTIVE') {
refreshDocs(sortField, currentPage, rowsPerPage);
}
}, [modalState, sortField, currentPage, rowsPerPage]);
return (
<div className="mt-8">
<div className="flex flex-col relative">
@@ -190,7 +190,7 @@ const Documents: React.FC<DocumentsProps> = ({
)}
{Array.isArray(currentDocuments) &&
currentDocuments.map((document, index) => (
<tr key={index}>
<tr key={index} className="text-nowrap font-normal">
<td>{document.name}</td>
<td>{document.date}</td>
<td>
@@ -248,18 +248,24 @@ const Documents: React.FC<DocumentsProps> = ({
</div>
)}
</div>
{/* Pagination component with props:
# Note: Every time the page changes,
the refreshDocs function is called with the updated page number and rows per page.
and reset cursor paginated query parameter to undefined.
*/}
<Pagination
currentPage={currentPage}
totalPages={totalPages}
rowsPerPage={rowsPerPage}
onPageChange={(page) => {
setCurrentPage(page);
refreshDocs(undefined, page, rowsPerPage);
refreshDocs(sortField, page, rowsPerPage); // Pass `true` to reset lastID if not using cursor
}}
onRowsPerPageChange={(rows) => {
console.log('Pagination - Rows per Page Change:', rows);
setRowsPerPage(rows);
setCurrentPage(1);
refreshDocs(undefined, 1, rows);
setCurrentPage(1); // Reset to page 1 on rows per page change
refreshDocs(sortField, 1, rows); // Reset lastID for fresh pagination
}}
/>
</div>
@@ -267,7 +273,7 @@ const Documents: React.FC<DocumentsProps> = ({
};
Documents.propTypes = {
documents: PropTypes.array.isRequired,
//documents: PropTypes.array.isRequired,
handleDeleteDocument: PropTypes.func.isRequired,
};

7
frontend/src/settings/index.tsx
View File

@@ -70,12 +70,7 @@ export default function Settings() {
case t('settings.general.label'):
return <General />;
case t('settings.documents.label'):
return (
<Documents
documents={documents}
handleDeleteDocument={handleDeleteClick}
/>
);
return <Documents handleDeleteDocument={handleDeleteClick} />;
case 'Widgets':
return (
<Widgets

27
frontend/src/upload/Upload.tsx
View File

@@ -166,7 +166,10 @@ function Upload({
dispatch(setSourceDocs(data));
dispatch(
setSelectedDocs(
data?.find((d) => d.type?.toLowerCase() === 'local'),
Array.isArray(data) &&
data?.find(
(d: Doc) => d.type?.toLowerCase() === 'local',
),
),
);
});
@@ -182,15 +185,21 @@ function Upload({
getDocs().then((data) => {
dispatch(setSourceDocs(data));
const docIds = new Set(
sourceDocs?.map((doc: Doc) => (doc.id ? doc.id : null)),
(Array.isArray(sourceDocs) &&
sourceDocs?.map((doc: Doc) =>
doc.id ? doc.id : null,
)) ||
[],
);
data?.map((updatedDoc: Doc) => {
if (updatedDoc.id && !docIds.has(updatedDoc.id)) {
//select the doc not present in the intersection of current Docs and fetched data
dispatch(setSelectedDocs(updatedDoc));
return;
}
});
if (data && Array.isArray(data.docs)) {
data.docs.map((updatedDoc: Doc) => {
if (updatedDoc.id && !docIds.has(updatedDoc.id)) {
// Select the doc not present in the intersection of current Docs and fetched data
dispatch(setSelectedDocs(updatedDoc));
return;
}
});
}
});
setProgress(
(progress) =>