diff --git a/application/app.py b/application/app.py index 357f00d3..dfce4970 100644 --- a/application/app.py +++ b/application/app.py @@ -136,6 +136,7 @@ def api_answer(): vectorstore = "" else: vectorstore = "" + print(vectorstore) # vectorstore = "outputs/inputs/" # loading the index and the store and the prompt template # Note if you have used other embeddings than OpenAI, you need to change the embeddings @@ -409,8 +410,11 @@ def delete_old(): if dirs[0] not in ['indexes', 'vectors']: return {"status": 'error'} path_clean = '/'.join(dirs) - shutil.rmtree(path) vectors_collection.delete_one({'location': path}) + try: + shutil.rmtree(path_clean) + except FileNotFoundError: + pass return {"status": 'ok'} # handling CORS diff --git a/frontend/.env.development b/frontend/.env.development index d9292fb3..701f5bac 100644 --- a/frontend/.env.development +++ b/frontend/.env.development @@ -1,2 +1,2 @@ # Please put appropriate value -VITE_API_HOST = https://docsapi.arc53.com \ No newline at end of file +VITE_API_HOST = http://localhost:5001 \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7e901646..f249b29e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1105,6 +1105,11 @@ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true }, + "attr-accept": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz", + "integrity": "sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg==" + }, "autoprefixer": { "version": "10.4.13", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz", @@ -2168,6 +2173,21 @@ "flat-cache": "^3.0.4" } }, + "file-selector": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.6.0.tgz", + "integrity": "sha512-QlZ5yJC0VxHxQQsQhXvBaC7VRJ2uaxTf+Tfpu4Z/OcVQJVpZO+DGU0rkoVW5ce2SccxugvpBJoMvUs59iILYdw==", + "requires": { + "tslib": "^2.4.0" + }, + "dependencies": { + "tslib": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", + "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + } + } + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -3088,8 +3108,7 @@ "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "dev": true + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" }, "object-hash": { "version": "3.0.0", @@ -3395,7 +3414,6 @@ "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", - "dev": true, "requires": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", @@ -3437,6 +3455,16 @@ "scheduler": "^0.23.0" } }, + "react-dropzone": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.2.3.tgz", + "integrity": "sha512-O3om8I+PkFKbxCukfIR3QAGftYXDZfOE2N1mr/7qebQJHs7U+/RSL/9xomJNpRg9kM5h9soQSdf0Gc7OHF5Fug==", + "requires": { + "attr-accept": "^2.2.2", + "file-selector": "^0.6.0", + "prop-types": "^15.8.1" + } + }, "react-is": { "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 4abaeba3..16ee4ffb 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -23,6 +23,7 @@ "@vercel/analytics": "^0.1.10", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-dropzone": "^14.2.3", "react-redux": "^8.0.5", "react-router-dom": "^6.8.1" }, diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index a2cedb03..f4b9eb96 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -2,11 +2,13 @@ import { useEffect, useRef, useState } from 'react'; import { NavLink } from 'react-router-dom'; import Arrow1 from './assets/arrow.svg'; import Arrow2 from './assets/dropdown-arrow.svg'; +import Exit from './assets/exit.svg'; import Message from './assets/message.svg'; import Hamburger from './assets/hamburger.svg'; import Key from './assets/key.svg'; import Info from './assets/info.svg'; import Link from './assets/link.svg'; +import UploadIcon from './assets/upload.svg'; import { ActiveState } from './models/misc'; import APIKeyModal from './preferences/APIKeyModal'; import SelectDocsModal from './preferences/SelectDocsModal'; @@ -19,6 +21,8 @@ import { setSelectedDocs, } from './preferences/preferenceSlice'; import { useOutsideAlerter } from './hooks'; +import Upload from './upload/Upload'; +import { Doc } from './preferences/preferenceApi'; export default function Navigation({ navState, @@ -42,7 +46,28 @@ export default function Navigation({ const [selectedDocsModalState, setSelectedDocsModalState] = useState(isSelectedDocsSet ? 'INACTIVE' : 'ACTIVE'); + const [uploadModalState, setUploadModalState] = + useState('INACTIVE'); + const navRef = useRef(null); + const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; + + const handleDeleteClick = (index: number, doc: Doc) => { + const docPath = 'indexes/' + 'local' + '/' + doc.name; + + fetch(`${apiHost}/api/delete_old?path=${docPath}`, { + method: 'GET', + }) + .then(() => { + // remove the image element from the DOM + const imageElement = document.querySelector( + `#img-${index}`, + ) as HTMLElement; + const parentElement = imageElement.parentNode as HTMLElement; + parentElement.parentNode?.removeChild(parentElement); + }) + .catch((error) => console.error(error)); + }; useOutsideAlerter( navRef, () => { @@ -109,7 +134,7 @@ export default function Navigation({
-
+
setIsDocsListOpen(!isDocsListOpen)} @@ -127,8 +152,13 @@ export default function Navigation({ } mr-3 w-3 transition-all`} />
+ setUploadModalState('ACTIVE')} + > {isDocsListOpen && ( -
+
{docs ? ( docs.map((doc, index) => { if (doc.model) { @@ -139,11 +169,23 @@ export default function Navigation({ dispatch(setSelectedDocs(doc)); setIsDocsListOpen(false); }} - className="h-10 w-full cursor-pointer border-x-2 border-b-2 hover:bg-gray-100" + className="flex h-10 w-full cursor-pointer items-center justify-between border-x-2 border-b-2 hover:bg-gray-100" > -

+

{doc.name} {doc.version}

+ {doc.location === 'local' ? ( + Exit { + event.stopPropagation(); + handleDeleteClick(index, doc); + }} + /> + ) : null}
); } @@ -153,6 +195,7 @@ export default function Navigation({

No default documentation.

)} + )
)}
@@ -222,6 +265,10 @@ export default function Navigation({ setModalState={setApiKeyModalState} isCancellable={isApiKeySet} /> + ); } diff --git a/frontend/src/assets/upload.svg b/frontend/src/assets/upload.svg new file mode 100644 index 00000000..6d7be211 --- /dev/null +++ b/frontend/src/assets/upload.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/conversation/conversationApi.ts b/frontend/src/conversation/conversationApi.ts index a0575682..c7320342 100644 --- a/frontend/src/conversation/conversationApi.ts +++ b/frontend/src/conversation/conversationApi.ts @@ -13,17 +13,20 @@ export function fetchAnswerApi( namePath = '.project'; } - const docPath = - selectedDocs.name === 'default' - ? 'default' - : selectedDocs.language + - '/' + - namePath + - '/' + - selectedDocs.version + - '/' + - selectedDocs.model + - '/'; + let docPath = 'default'; + if (selectedDocs.location === 'local') { + docPath = 'local' + '/' + selectedDocs.name + '/'; + } else if (selectedDocs.location === 'remote') { + docPath = + selectedDocs.language + + '/' + + namePath + + '/' + + selectedDocs.version + + '/' + + selectedDocs.model + + '/'; + } return fetch(apiHost + '/api/answer', { method: 'POST', diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index d4d5979c..b9646a15 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -1,5 +1,6 @@ // not all properties in Doc are going to be present. Make some optional export type Doc = { + location: string; name: string; language: string; version: string; @@ -13,9 +14,10 @@ export type Doc = { //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later. export async function getDocs(): Promise { try { - const response = await fetch( - 'https://d3dg1063dc54p9.cloudfront.net/combined.json', - ); + const apiHost = + import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; + + const response = await fetch(apiHost + '/api/combine'); const data = await response.json(); const docs: Doc[] = []; @@ -52,17 +54,13 @@ export function setLocalRecentDocs(doc: Doc): void { namePath = '.project'; } - const docPath = - doc.name === 'default' - ? 'default' - : doc.language + - '/' + - namePath + - '/' + - doc.version + - '/' + - doc.model + - '/'; + let docPath = 'default'; + if (doc.location === 'local') { + docPath = 'local' + '/' + doc.name + '/'; + } else if (doc.location === 'remote') { + docPath = + doc.language + '/' + namePath + '/' + doc.version + '/' + doc.model + '/'; + } const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; fetch(apiHost + '/api/docs_check', { method: 'POST', diff --git a/frontend/src/store.ts b/frontend/src/store.ts index 6bea07b2..dd0a2c9c 100644 --- a/frontend/src/store.ts +++ b/frontend/src/store.ts @@ -15,6 +15,7 @@ const store = configureStore({ selectedDocs: doc !== null ? JSON.parse(doc) : null, sourceDocs: [ { + location: '', language: '', name: 'default', version: '', diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx new file mode 100644 index 00000000..e404bd12 --- /dev/null +++ b/frontend/src/upload/Upload.tsx @@ -0,0 +1,201 @@ +import React from 'react'; +import { useCallback, useEffect, useState } from 'react'; +import { useDropzone } from 'react-dropzone'; +import { useDispatch } from 'react-redux'; +import { ActiveState } from '../models/misc'; +import { getDocs } from '../preferences/preferenceApi'; +import { setSourceDocs } from '../preferences/preferenceSlice'; + +export default function Upload({ + modalState, + setModalState, +}: { + modalState: ActiveState; + setModalState: (state: ActiveState) => void; +}) { + const [docName, setDocName] = useState(''); + const [files, setfiles] = useState([]); + const [progress, setProgress] = useState<{ + type: 'UPLOAD' | 'TRAINIING'; + percentage: number; + taskId?: string; + }>(); + + function Progress({ + title, + isCancellable = false, + }: { + title: string; + isCancellable?: boolean; + }) { + return ( +
+

{title}...

+

This may take several minutes

+

{progress?.percentage || 0}%

+
+
+
+
+ +
+ ); + } + + function UploadProgress() { + return ; + } + + function TrainingProgress() { + const dispatch = useDispatch(); + useEffect(() => { + (progress?.percentage ?? 0) < 100 && + setTimeout(() => { + const apiHost = import.meta.env.VITE_API_HOST; + fetch(`${apiHost}/api/task_status?task_id=${progress?.taskId}`) + .then((data) => data.json()) + .then((data) => { + if (data.status == 'SUCCESS') { + getDocs().then((data) => dispatch(setSourceDocs(data))); + setProgress( + (progress) => progress && { ...progress, percentage: 100 }, + ); + } else { + setProgress( + (progress) => + progress && { + ...progress, + percentage: data.result.current, + }, + ); + } + }); + }, 5000); + }, [progress, dispatch]); + return ( + + ); + } + + const onDrop = useCallback((acceptedFiles: File[]) => { + setfiles(acceptedFiles); + setDocName(acceptedFiles[0]?.name); + }, []); + + const doNothing = () => undefined; + + const uploadFile = () => { + const formData = new FormData(); + files.forEach((file) => { + formData.append('file', file); + }); + formData.append('name', docName); + formData.append('user', 'local'); + const apiHost = import.meta.env.VITE_API_HOST; + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener('progress', (event) => { + const progress = +((event.loaded / event.total) * 100).toFixed(2); + setProgress({ type: 'UPLOAD', percentage: progress }); + }); + xhr.onload = () => { + const { task_id } = JSON.parse(xhr.responseText); + setProgress({ type: 'TRAINIING', percentage: 0, taskId: task_id }); + }; + xhr.open('POST', `${apiHost + '/api/upload'}`); + xhr.send(formData); + }; + + const { getRootProps, getInputProps, isDragActive } = useDropzone({ + onDrop, + multiple: true, + onDragEnter: doNothing, + onDragOver: doNothing, + onDragLeave: doNothing, + }); + + let view; + if (progress?.type === 'UPLOAD') { + view = ; + } else if (progress?.type === 'TRAINIING') { + view = ; + } else { + view = ( + <> +

Upload New Documentation

+ setDocName(e.target.value)} + > +
+ Name +
+
+ + + Choose Files + +
+
+

Uploaded Files

+ {files.map((file) => ( +

+ {file.name} +

+ ))} + {files.length === 0 &&

None

} +
+
+ + +
+ + ); + } + + return ( +
+
+ {view} +
+
+ ); +} +// TODO: sanitize all inputs diff --git a/frontend/tailwind.config.cjs b/frontend/tailwind.config.cjs index 2c44007a..64fb76f2 100644 --- a/frontend/tailwind.config.cjs +++ b/frontend/tailwind.config.cjs @@ -16,10 +16,16 @@ module.exports = { 'gray-2000': 'rgba(0, 0, 0, 0.5)', 'gray-3000': 'rgba(243, 243, 243, 1)', 'gray-4000': '#949494', + 'gray-5000': '#BBBBBB', + 'gray-6000': '#757575', 'red-1000': 'rgb(254, 202, 202)', 'red-2000': '#F44336', 'red-3000': '#621B16', 'blue-1000': '#7D54D1', + 'blue-2000': '#002B49', + 'blue-3000': '#4B02E2', + 'blue-4000': 'rgba(0, 125, 255, 0.36)', + 'blue-5000': 'rgba(0, 125, 255)', }, }, },