LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-30 00:53:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/analytics-and-logs

Browse Source
This commit is contained in:
Siddhant Rai
2024-09-11 17:58:04 +05:30
committed by GitHub
parent bea0bbfcdb c12d7a8d82
commit 72e68a163c
51 changed files with 1116 additions and 1498 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
frontend/src/Navigation.tsx
View File

@@ -24,9 +24,9 @@ import ConversationTile from './conversation/ConversationTile';
import { useDarkTheme, useMediaQuery, useOutsideAlerter } from './hooks';
import useDefaultDocument from './hooks/useDefaultDocument';
import DeleteConvModal from './modals/DeleteConvModal';
import { ActiveState } from './models/misc';
import { ActiveState, Doc } from './models/misc';
import APIKeyModal from './preferences/APIKeyModal';
import { Doc, getConversations, getDocs } from './preferences/preferenceApi';
import { getConversations, getDocs } from './preferences/preferenceApi';
import {
selectApiKeyStatus,
selectConversationId,
@@ -124,10 +124,8 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
};
const handleDeleteClick = (doc: Doc) => {
const docPath = `indexes/local/${doc.name}`;
userService
.deletePath(docPath)
.deletePath(doc.id ?? '')
.then(() => {
return getDocs();
})

2
frontend/src/api/endpoints.ts
View File

@@ -10,7 +10,7 @@ const endpoints = {
DELETE_PROMPT: '/api/delete_prompt',
UPDATE_PROMPT: '/api/update_prompt',
SINGLE_PROMPT: (id: string) => `/api/get_single_prompt?id=${id}`,
DELETE_PATH: (docPath: string) => `/api/delete_old?path=${docPath}`,
DELETE_PATH: (docPath: string) => `/api/delete_old?source_id=${docPath}`,
TASK_STATUS: (task_id: string) => `/api/task_status?task_id=${task_id}`,
MESSAGE_ANALYTICS: '/api/get_message_analytics',
TOKEN_ANALYTICS: '/api/get_token_analytics',

1
frontend/src/components/Dropdown.tsx
View File

@@ -27,6 +27,7 @@ function Dropdown({
| string
| { label: string; value: string }
| { value: number; description: string }
| { name: string; id: string; type: string }
| null;
onSelect:
| ((value: string) => void)

5
frontend/src/components/SourceDropdown.tsx
View File

@@ -1,7 +1,7 @@
import React from 'react';
import Trash from '../assets/trash.svg';
import Arrow2 from '../assets/dropdown-arrow.svg';
import { Doc } from '../preferences/preferenceApi';
import { Doc } from '../models/misc';
import { useDispatch } from 'react-redux';
import { useTranslation } from 'react-i18next';
type Props = {
@@ -63,9 +63,6 @@ function SourceDropdown({
<p className="max-w-3/4 truncate whitespace-nowrap">
{selectedDocs?.name || 'None'}
</p>
<p className="flex flex-col items-center justify-center">
{selectedDocs?.version}
</p>
</div>
</span>
<img

41
frontend/src/conversation/Conversation.tsx
View File

@@ -1,4 +1,4 @@
import React, { Fragment, useEffect, useRef, useState } from 'react';
import { Fragment, useEffect, useRef, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useDispatch, useSelector } from 'react-redux';
@@ -30,7 +30,7 @@ export default function Conversation() {
const status = useSelector(selectStatus);
const conversationId = useSelector(selectConversationId);
const dispatch = useDispatch<AppDispatch>();
const endMessageRef = useRef<HTMLDivElement>(null);
const conversationRef = useRef<HTMLDivElement>(null);
const inputRef = useRef<HTMLTextAreaElement>(null);
const [isDarkTheme] = useDarkTheme();
const [hasScrolledToLast, setHasScrolledToLast] = useState(true);
@@ -58,26 +58,6 @@ export default function Conversation() {
fetchStream.current && fetchStream.current.abort();
}, [conversationId]);
useEffect(() => {
const observerCallback: IntersectionObserverCallback = (entries) => {
entries.forEach((entry) => {
setHasScrolledToLast(entry.isIntersecting);
});
};
const observer = new IntersectionObserver(observerCallback, {
root: null,
threshold: [1, 0.8],
});
if (endMessageRef.current) {
observer.observe(endMessageRef.current);
}
return () => {
observer.disconnect();
};
}, [endMessageRef.current]);
useEffect(() => {
if (queries.length) {
queries[queries.length - 1].error && setLastQueryReturnedErr(true);
@@ -86,10 +66,16 @@ export default function Conversation() {
}, [queries[queries.length - 1]]);
const scrollIntoView = () => {
endMessageRef?.current?.scrollIntoView({
behavior: 'smooth',
block: 'start',
});
if (!conversationRef?.current || eventInterrupt) return;
if (status === 'idle' || !queries[queries.length - 1].response) {
conversationRef.current.scrollTo({
behavior: 'smooth',
top: conversationRef.current.scrollHeight,
});
} else {
conversationRef.current.scrollTop = conversationRef.current.scrollHeight;
}
};
const handleQuestion = ({
@@ -143,7 +129,6 @@ export default function Conversation() {
if (query.response) {
responseView = (
<ConversationBubble
ref={endMessageRef}
className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'}`}
key={`${index}ANSWER`}
message={query.response}
@@ -176,7 +161,6 @@ export default function Conversation() {
);
responseView = (
<ConversationBubble
ref={endMessageRef}
className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'} `}
key={`${index}ERROR`}
message={query.error}
@@ -234,6 +218,7 @@ export default function Conversation() {
</>
)}
<div
ref={conversationRef}
onWheel={handleUserInterruption}
onTouchMove={handleUserInterruption}
className="flex h-[90%] w-full flex-1 justify-center overflow-y-auto p-4 md:h-[83vh]"

5
frontend/src/conversation/ConversationBubble.tsx
View File

@@ -250,7 +250,10 @@ const ConversationBubble = forwardRef<
</div>
</div>
) : (
<code className={className ? className : ''} {...props}>
<code
className={className ? className : 'whitespace-pre-line'}
{...props}
>
{children}
</code>
);

86
frontend/src/conversation/conversationHandlers.ts
View File

@@ -1,32 +1,6 @@
import conversationService from '../api/services/conversationService';
import { Doc } from '../preferences/preferenceApi';
import { Answer, FEEDBACK } from './conversationModels';
function getDocPath(selectedDocs: Doc | null): string {
let docPath = 'default';
if (selectedDocs) {
let namePath = selectedDocs.name;
if (selectedDocs.language === namePath) {
namePath = '.project';
}
if (selectedDocs.location === 'local') {
docPath = 'local' + '/' + selectedDocs.name + '/';
} else if (selectedDocs.location === 'remote') {
docPath =
selectedDocs.language +
'/' +
namePath +
'/' +
selectedDocs.version +
'/' +
selectedDocs.model +
'/';
} else if (selectedDocs.location === 'custom') {
docPath = selectedDocs.docLink;
}
}
return docPath;
}
import { Doc } from '../models/misc';
import { Answer, FEEDBACK, RetrievalPayload } from './conversationModels';
export function handleFetchAnswer(
question: string,
@@ -54,23 +28,22 @@ export function handleFetchAnswer(
title: any;
}
> {
const docPath = getDocPath(selectedDocs);
history = history.map((item) => {
return { prompt: item.prompt, response: item.response };
});
const payload: RetrievalPayload = {
question: question,
history: JSON.stringify(history),
conversation_id: conversationId,
prompt_id: promptId,
chunks: chunks,
token_limit: token_limit,
};
if (selectedDocs && 'id' in selectedDocs)
payload.active_docs = selectedDocs.id as string;
payload.retriever = selectedDocs?.retriever as string;
return conversationService
.answer(
{
question: question,
history: history,
active_docs: docPath,
conversation_id: conversationId,
prompt_id: promptId,
chunks: chunks,
token_limit: token_limit,
},
signal,
)
.answer(payload, signal)
.then((response) => {
if (response.ok) {
return response.json();
@@ -101,16 +74,27 @@ export function handleFetchAnswerSteaming(
token_limit: number,
onEvent: (event: MessageEvent) => void,
): Promise<Answer> {
const docPath = getDocPath(selectedDocs);
history = history.map((item) => {
return { prompt: item.prompt, response: item.response };
});
const payload: RetrievalPayload = {
question: question,
history: JSON.stringify(history),
conversation_id: conversationId,
prompt_id: promptId,
chunks: chunks,
token_limit: token_limit,
};
if (selectedDocs && 'id' in selectedDocs)
payload.active_docs = selectedDocs.id as string;
payload.retriever = selectedDocs?.retriever as string;
return new Promise<Answer>((resolve, reject) => {
conversationService
.answerStream(
{
question: question,
active_docs: docPath,
active_docs: selectedDocs?.id as string,
history: JSON.stringify(history),
conversation_id: conversationId,
prompt_id: promptId,
@@ -176,11 +160,23 @@ export function handleSearch(
chunks: string,
token_limit: number,
) {
const docPath = getDocPath(selectedDocs);
history = history.map((item) => {
return { prompt: item.prompt, response: item.response };
});
const payload: RetrievalPayload = {
question: question,
history: JSON.stringify(history),
conversation_id: conversation_id,
chunks: chunks,
token_limit: token_limit,
};
if (selectedDocs && 'id' in selectedDocs)
payload.active_docs = selectedDocs.id as string;
payload.retriever = selectedDocs?.retriever as string;
return conversationService
.search({
question: question,
active_docs: docPath,
active_docs: selectedDocs?.id as string,
conversation_id,
history,
chunks: chunks,

10
frontend/src/conversation/conversationModels.ts
View File

@@ -31,3 +31,13 @@ export interface Query {
conversationId?: string | null;
title?: string | null;
}
export interface RetrievalPayload {
question: string;
active_docs?: string;
retriever?: string;
history: string;
conversation_id: string | null;
prompt_id?: string | null;
chunks: string;
token_limit: number;
}

3
frontend/src/hooks/useDefaultDocument.ts
View File

@@ -1,7 +1,8 @@
import React from 'react';
import { useDispatch, useSelector } from 'react-redux';
import { Doc, getDocs } from '../preferences/preferenceApi';
import { getDocs } from '../preferences/preferenceApi';
import { Doc } from '../models/misc';
import {
selectSelectedDocs,
setSelectedDocs,

70
frontend/src/modals/CreateAPIKeyModal.tsx
View File

@@ -22,8 +22,9 @@ export default function CreateAPIKeyModal({
const [APIKeyName, setAPIKeyName] = React.useState<string>('');
const [sourcePath, setSourcePath] = React.useState<{
label: string;
value: string;
name: string;
id: string;
type: string;
} | null>(null);
const [prompt, setPrompt] = React.useState<{
name: string;
@@ -41,27 +42,17 @@ export default function CreateAPIKeyModal({
? docs
.filter((doc) => doc.model === embeddingsName)
.map((doc: Doc) => {
let namePath = doc.name;
if (doc.language === namePath) {
namePath = '.project';
}
let docPath = 'default';
if (doc.location === 'local') {
docPath = 'local' + '/' + doc.name + '/';
} else if (doc.location === 'remote') {
docPath =
doc.language +
'/' +
namePath +
'/' +
doc.version +
'/' +
doc.model +
'/';
if ('id' in doc) {
return {
name: doc.name,
id: doc.id as string,
type: 'local',
};
}
return {
label: doc.name,
value: docPath,
name: doc.name,
id: doc.id ?? 'default',
type: doc.type ?? 'default',
};
})
: [];
@@ -107,9 +98,14 @@ export default function CreateAPIKeyModal({
<Dropdown
placeholder={t('modals.createAPIKey.sourceDoc')}
selectedValue={sourcePath}
onSelect={(selection: { label: string; value: string }) =>
setSourcePath(selection)
}
onSelect={(selection: {
name: string;
id: string;
type: string;
}) => {
setSourcePath(selection);
console.log(selection);
}}
options={extractDocPaths()}
size="w-full"
rounded="xl"
@@ -142,16 +138,22 @@ export default function CreateAPIKeyModal({
</div>
<button
disabled={!sourcePath || APIKeyName.length === 0 || !prompt}
onClick={() =>
sourcePath &&
prompt &&
createAPIKey({
name: APIKeyName,
source: sourcePath.value,
prompt_id: prompt.id,
chunks: chunk,
})
}
onClick={() => {
if (sourcePath && prompt) {
const payload: any = {
name: APIKeyName,
prompt_id: prompt.id,
chunks: chunk,
};
if (sourcePath.type === 'default') {
payload.retriever = sourcePath.id;
}
if (sourcePath.type === 'local') {
payload.source = sourcePath.id;
}
createAPIKey(payload);
}
}}
className="float-right mt-4 rounded-full bg-purple-30 px-5 py-2 text-sm text-white hover:bg-[#6F3FD1] disabled:opacity-50"
>
{t('modals.createAPIKey.create')}

20
frontend/src/modals/ShareConversationModal.tsx
View File

@@ -46,27 +46,9 @@ export const ShareConversationModal = ({
? docs
.filter((doc) => doc.model === embeddingsName)
.map((doc: Doc) => {
let namePath = doc.name;
if (doc.language === namePath) {
namePath = '.project';
}
let docPath = 'default';
if (doc.location === 'local') {
docPath = 'local' + '/' + doc.name + '/';
} else if (doc.location === 'remote') {
docPath =
doc.language +
'/' +
namePath +
'/' +
doc.version +
'/' +
doc.model +
'/';
}
return {
label: doc.name,
value: docPath,
value: doc.id ?? 'default',
};
})
: [];

9
frontend/src/models/misc.ts
View File

@@ -4,16 +4,13 @@ export type User = {
avatar: string;
};
export type Doc = {
location: string;
id?: string;
name: string;
language: string;
version: string;
description: string;
fullName: string;
date: string;
docLink: string;
model: string;
tokens?: string;
type?: string;
retriever?: string;
};
export type PromptProps = {

23
frontend/src/preferences/preferenceApi.ts
View File

@@ -1,18 +1,6 @@
import conversationService from '../api/services/conversationService';
import userService from '../api/services/userService';
// not all properties in Doc are going to be present. Make some optional
export type Doc = {
location: string;
name: string;
language: string;
version: string;
description: string;
fullName: string;
date: string;
docLink: string;
model: string;
};
import { Doc } from '../models/misc';
//Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
export async function getDocs(): Promise<Doc[] | null> {
@@ -78,17 +66,10 @@ export function setLocalPrompt(prompt: string): void {
export function setLocalRecentDocs(doc: Doc | null): void {
localStorage.setItem('DocsGPTRecentDocs', JSON.stringify(doc));
let namePath = doc?.name;
if (doc?.language === namePath) {
namePath = '.project';
}
let docPath = 'default';
if (doc?.location === 'local') {
if (doc?.type === 'local') {
docPath = 'local' + '/' + doc.name + '/';
} else if (doc?.location === 'remote') {
docPath =
doc.language + '/' + namePath + '/' + doc.version + '/' + doc.model + '/';
}
userService
.checkDocs({

12
frontend/src/preferences/preferenceSlice.ts
View File

@@ -4,9 +4,9 @@ import {
createSlice,
isAnyOf,
} from '@reduxjs/toolkit';
import { Doc, setLocalApiKey, setLocalRecentDocs } from './preferenceApi';
import { setLocalApiKey, setLocalRecentDocs } from './preferenceApi';
import { RootState } from '../store';
import { ActiveState } from '../models/misc';
import { ActiveState, Doc } from '../models/misc';
interface Preference {
apiKey: string;
@@ -25,15 +25,13 @@ const initialState: Preference = {
chunks: '2',
token_limit: 2000,
selectedDocs: {
id: 'default',
name: 'default',
language: 'default',
location: 'default',
version: 'default',
description: 'default',
fullName: 'default',
type: 'remote',
date: 'default',
docLink: 'default',
model: 'openai_text-embedding-ada-002',
retriever: 'classic',
} as Doc,
sourceDocs: null,
conversations: null,

3
frontend/src/settings/APIKeys.tsx
View File

@@ -47,7 +47,8 @@ export default function APIKeys() {
const handleCreateKey = (payload: {
name: string;
source: string;
source?: string;
retriever?: string;
prompt_id: string;
chunks: string;
}) => {

6
frontend/src/settings/Documents.tsx
View File

@@ -61,12 +61,10 @@ const Documents: React.FC<DocumentsProps> = ({
{document.tokens ? formatTokens(+document.tokens) : ''}
</td>
<td className="border-r border-t px-4 py-2">
{document.location === 'remote'
? 'Pre-loaded'
: 'Private'}
{document.type === 'remote' ? 'Pre-loaded' : 'Private'}
</td>
<td className="border-t px-4 py-2">
{document.location !== 'remote' && (
{document.type !== 'remote' && (
<img
src={Trash}
alt="Delete"

5
frontend/src/settings/index.tsx
View File

@@ -6,7 +6,7 @@ import userService from '../api/services/userService';
import ArrowLeft from '../assets/arrow-left.svg';
import ArrowRight from '../assets/arrow-right.svg';
import i18n from '../locale/i18n';
import { Doc } from '../preferences/preferenceApi';
import { Doc } from '../models/misc';
import {
selectSourceDocs,
setSourceDocs,
@@ -39,9 +39,8 @@ export default function Settings() {
};
const handleDeleteClick = (index: number, doc: Doc) => {
const docPath = 'indexes/' + 'local' + '/' + doc.name;
userService
.deletePath(docPath)
.deletePath(doc.id ?? '')
.then((response) => {
if (response.ok && documents) {
const updatedDocuments = [

9
frontend/src/store.ts
View File

@@ -26,15 +26,12 @@ const store = configureStore({
conversations: null,
sourceDocs: [
{
location: '',
language: '',
name: 'default',
version: '',
date: '',
description: '',
docLink: '',
fullName: '',
model: '1.0',
type: 'remote',
id: 'default',
retriever: 'clasic',
},
],
modalState: 'INACTIVE',

4
frontend/src/upload/Upload.tsx
View File

@@ -120,7 +120,7 @@ function Upload({
dispatch(setSourceDocs(data));
dispatch(
setSelectedDocs(
data?.find((d) => d.location.toLowerCase() === 'local'),
data?.find((d) => d.type?.toLowerCase() === 'local'),
),
);
});
@@ -137,7 +137,7 @@ function Upload({
dispatch(setSourceDocs(data));
dispatch(
setSelectedDocs(
data?.find((d) => d.location.toLowerCase() === 'local'),
data?.find((d) => d.type?.toLowerCase() === 'local'),
),
);
});