From 51c42790b7b737fdb205856d7dc4a29674e24cd2 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Mon, 9 Sep 2024 16:20:07 +0530 Subject: [PATCH 1/4] widget: add option to collect feedback --- application/api/user/routes.py | 94 +++++----------- .../react-widget/src/assets/dislike.svg | 4 + extensions/react-widget/src/assets/like.svg | 4 + .../src/components/DocsGPTWidget.tsx | 100 ++++++++++++++++-- .../react-widget/src/requests/streamingApi.ts | 29 ++++- extensions/react-widget/src/types/index.ts | 1 + 6 files changed, 148 insertions(+), 84 deletions(-) create mode 100644 extensions/react-widget/src/assets/dislike.svg create mode 100644 extensions/react-widget/src/assets/like.svg diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 91b90d6a..c49b43fd 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -25,9 +25,7 @@ shared_conversations_collections = db["shared_conversations"] user = Blueprint("user", __name__) -current_dir = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -) +current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @user.route("/api/delete_conversation", methods=["POST"]) @@ -57,9 +55,7 @@ def get_conversations(): conversations = conversations_collection.find().sort("date", -1).limit(30) list_conversations = [] for conversation in conversations: - list_conversations.append( - {"id": str(conversation["_id"]), "name": conversation["name"]} - ) + list_conversations.append({"id": str(conversation["_id"]), "name": conversation["name"]}) # list_conversations = [{"id": "default", "name": "default"}, {"id": "jeff", "name": "jeff"}] @@ -90,14 +86,10 @@ def api_feedback(): question = data["question"] answer = data["answer"] feedback = data["feedback"] - - feedback_collection.insert_one( - { - "question": question, - "answer": answer, - "feedback": feedback, - } - ) + new_doc = {"question": question, "answer": answer, "feedback": feedback} + if "api_key" in data: + new_doc["api_key"] = data["api_key"] + feedback_collection.insert_one(new_doc) return {"status": "ok"} @@ -138,9 +130,7 @@ def delete_old(): except FileNotFoundError: pass else: - vetorstore = VectorCreator.create_vectorstore( - settings.VECTOR_STORE, path=os.path.join(current_dir, path_clean) - ) + vetorstore = VectorCreator.create_vectorstore(settings.VECTOR_STORE, path=os.path.join(current_dir, path_clean)) vetorstore.delete_index() return {"status": "ok"} @@ -175,9 +165,7 @@ def upload_file(): file.save(os.path.join(temp_dir, filename)) # Use shutil.make_archive to zip the temp directory - zip_path = shutil.make_archive( - base_name=os.path.join(save_dir, job_name), format="zip", root_dir=temp_dir - ) + zip_path = shutil.make_archive(base_name=os.path.join(save_dir, job_name), format="zip", root_dir=temp_dir) final_filename = os.path.basename(zip_path) # Clean up the temporary directory after zipping @@ -219,9 +207,7 @@ def upload_remote(): source_data = request.form["data"] if source_data: - task = ingest_remote.delay( - source_data=source_data, job_name=job_name, user=user, loader=source - ) + task = ingest_remote.delay(source_data=source_data, job_name=job_name, user=user, loader=source) task_id = task.id return {"status": "ok", "task_id": task_id} else: @@ -277,9 +263,7 @@ def combined_json(): } ) if settings.VECTOR_STORE == "faiss": - data_remote = requests.get( - "https://d3dg1063dc54p9.cloudfront.net/combined.json" - ).json() + data_remote = requests.get("https://d3dg1063dc54p9.cloudfront.net/combined.json").json() for index in data_remote: index["location"] = "remote" data.append(index) @@ -382,9 +366,7 @@ def get_prompts(): list_prompts.append({"id": "creative", "name": "creative", "type": "public"}) list_prompts.append({"id": "strict", "name": "strict", "type": "public"}) for prompt in prompts: - list_prompts.append( - {"id": str(prompt["_id"]), "name": prompt["name"], "type": "private"} - ) + list_prompts.append({"id": str(prompt["_id"]), "name": prompt["name"], "type": "private"}) return jsonify(list_prompts) @@ -393,21 +375,15 @@ def get_prompts(): def get_single_prompt(): prompt_id = request.args.get("id") if prompt_id == "default": - with open( - os.path.join(current_dir, "prompts", "chat_combine_default.txt"), "r" - ) as f: + with open(os.path.join(current_dir, "prompts", "chat_combine_default.txt"), "r") as f: chat_combine_template = f.read() return jsonify({"content": chat_combine_template}) elif prompt_id == "creative": - with open( - os.path.join(current_dir, "prompts", "chat_combine_creative.txt"), "r" - ) as f: + with open(os.path.join(current_dir, "prompts", "chat_combine_creative.txt"), "r") as f: chat_reduce_creative = f.read() return jsonify({"content": chat_reduce_creative}) elif prompt_id == "strict": - with open( - os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r" - ) as f: + with open(os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r") as f: chat_reduce_strict = f.read() return jsonify({"content": chat_reduce_strict}) @@ -436,9 +412,7 @@ def update_prompt_name(): # check if name is null if name == "": return {"status": "error"} - prompts_collection.update_one( - {"_id": ObjectId(id)}, {"$set": {"name": name, "content": content}} - ) + prompts_collection.update_one({"_id": ObjectId(id)}, {"$set": {"name": name, "content": content}}) return {"status": "ok"} @@ -506,9 +480,7 @@ def share_conversation(): conversation_id = data["conversation_id"] isPromptable = request.args.get("isPromptable").lower() == "true" - conversation = conversations_collection.find_one( - {"_id": ObjectId(conversation_id)} - ) + conversation = conversations_collection.find_one({"_id": ObjectId(conversation_id)}) current_n_queries = len(conversation["queries"]) ##generate binary representation of uuid @@ -533,9 +505,7 @@ def share_conversation(): api_uuid = pre_existing_api_document["key"] pre_existing = shared_conversations_collections.find_one( { - "conversation_id": DBRef( - "conversations", ObjectId(conversation_id) - ), + "conversation_id": DBRef("conversations", ObjectId(conversation_id)), "isPromptable": isPromptable, "first_n_queries": current_n_queries, "user": user, @@ -566,9 +536,7 @@ def share_conversation(): "api_key": api_uuid, } ) - return jsonify( - {"success": True, "identifier": str(explicit_binary.as_uuid())} - ) + return jsonify({"success": True, "identifier": str(explicit_binary.as_uuid())}) else: api_key_collection.insert_one( { @@ -595,9 +563,7 @@ def share_conversation(): ) ## Identifier as route parameter in frontend return ( - jsonify( - {"success": True, "identifier": str(explicit_binary.as_uuid())} - ), + jsonify({"success": True, "identifier": str(explicit_binary.as_uuid())}), 201, ) @@ -612,9 +578,7 @@ def share_conversation(): ) if pre_existing is not None: return ( - jsonify( - {"success": True, "identifier": str(pre_existing["uuid"].as_uuid())} - ), + jsonify({"success": True, "identifier": str(pre_existing["uuid"].as_uuid())}), 200, ) else: @@ -632,9 +596,7 @@ def share_conversation(): ) ## Identifier as route parameter in frontend return ( - jsonify( - {"success": True, "identifier": str(explicit_binary.as_uuid())} - ), + jsonify({"success": True, "identifier": str(explicit_binary.as_uuid())}), 201, ) except Exception as err: @@ -646,16 +608,10 @@ def share_conversation(): @user.route("/api/shared_conversation/", methods=["GET"]) def get_publicly_shared_conversations(identifier: str): try: - query_uuid = Binary.from_uuid( - uuid.UUID(identifier), UuidRepresentation.STANDARD - ) + query_uuid = Binary.from_uuid(uuid.UUID(identifier), UuidRepresentation.STANDARD) shared = shared_conversations_collections.find_one({"uuid": query_uuid}) conversation_queries = [] - if ( - shared - and "conversation_id" in shared - and isinstance(shared["conversation_id"], DBRef) - ): + if shared and "conversation_id" in shared and isinstance(shared["conversation_id"], DBRef): # Resolve the DBRef conversation_ref = shared["conversation_id"] conversation = db.dereference(conversation_ref) @@ -669,9 +625,7 @@ def get_publicly_shared_conversations(identifier: str): ), 404, ) - conversation_queries = conversation["queries"][ - : (shared["first_n_queries"]) - ] + conversation_queries = conversation["queries"][: (shared["first_n_queries"])] for query in conversation_queries: query.pop("sources") ## avoid exposing sources else: diff --git a/extensions/react-widget/src/assets/dislike.svg b/extensions/react-widget/src/assets/dislike.svg new file mode 100644 index 00000000..ec1d24c2 --- /dev/null +++ b/extensions/react-widget/src/assets/dislike.svg @@ -0,0 +1,4 @@ + + + + diff --git a/extensions/react-widget/src/assets/like.svg b/extensions/react-widget/src/assets/like.svg new file mode 100644 index 00000000..c49604ed --- /dev/null +++ b/extensions/react-widget/src/assets/like.svg @@ -0,0 +1,4 @@ + + + + diff --git a/extensions/react-widget/src/components/DocsGPTWidget.tsx b/extensions/react-widget/src/components/DocsGPTWidget.tsx index bc6adb6e..7b0f5bb3 100644 --- a/extensions/react-widget/src/components/DocsGPTWidget.tsx +++ b/extensions/react-widget/src/components/DocsGPTWidget.tsx @@ -1,11 +1,13 @@ "use client"; -import React from 'react' +import React, { useRef } from 'react' import DOMPurify from 'dompurify'; import styled, { keyframes, createGlobalStyle } from 'styled-components'; import { PaperPlaneIcon, RocketIcon, ExclamationTriangleIcon, Cross2Icon } from '@radix-ui/react-icons'; -import { MESSAGE_TYPE, Query, Status, WidgetProps } from '../types/index'; -import { fetchAnswerStreaming } from '../requests/streamingApi'; +import { FEEDBACK, MESSAGE_TYPE, Query, Status, WidgetProps } from '../types/index'; +import { fetchAnswerStreaming, sendFeedback } from '../requests/streamingApi'; import { ThemeProvider } from 'styled-components'; +import Like from "../assets/like.svg" +import Dislike from "../assets/dislike.svg" import MarkdownIt from 'markdown-it'; const themes = { dark: { @@ -63,6 +65,14 @@ const GlobalStyles = createGlobalStyle` background-color: #646464; color: #fff !important; } +.feedback > .selected{ + fill: #fff; + display: block !important; +} +.feedback > .default{ + fill:#000; + display:block; +} `; const Overlay = styled.div` position: fixed; @@ -197,10 +207,15 @@ const Conversation = styled.div<{ size: string }>` `; const MessageBubble = styled.div<{ type: MESSAGE_TYPE }>` - display: flex; + display: block; font-size: 16px; - justify-content: ${props => props.type === 'QUESTION' ? 'flex-end' : 'flex-start'}; - margin: 0.5rem; + position: relative; + width: 100%; + float: right; + margin: 0rem; + &:hover .feedback-icons { + visibility: visible !important; + } `; const Message = styled.div<{ type: MESSAGE_TYPE }>` background: ${props => props.type === 'QUESTION' ? @@ -208,6 +223,7 @@ const Message = styled.div<{ type: MESSAGE_TYPE }>` props.theme.secondary.bg}; color: ${props => props.type === 'ANSWER' ? props.theme.primary.text : '#fff'}; border: none; + float: ${props => props.type === 'QUESTION' ? 'right' : 'left'}; max-width: ${props => props.type === 'ANSWER' ? '100%' : '80'}; overflow: auto; margin: 4px; @@ -315,6 +331,14 @@ const HeroDescription = styled.p` font-size: 14px; line-height: 1.5; `; +const Feedback = styled.div` + background-color: transparent; + font-weight: normal; + gap: 12px; + display: flex; + padding: 6px; + clear: both; +` const Hero = ({ title, description, theme }: { title: string, description: string, theme: string }) => { return ( <> @@ -335,8 +359,8 @@ const Hero = ({ title, description, theme }: { title: string, description: strin ); }; export const DocsGPTWidget = ({ - apiHost = 'https://gptcloud.arc53.com', - apiKey = '82962c9a-aa77-4152-94e5-a4f84fd44c6a', + apiHost = 'http://127.0.0.1:7091', + apiKey = "220a4876-014d-47c8-996b-936fbefd3a22",//'82962c9a-aa77-4152-94e5-a4f84fd44c6a', avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png', title = 'Get AI assistance', description = 'DocsGPT\'s AI Chatbot is here to help', @@ -345,7 +369,8 @@ export const DocsGPTWidget = ({ size = 'small', theme = 'dark', buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/message.svg', - buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)' + buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)', + collectFeedback = false }: WidgetProps) => { const [prompt, setPrompt] = React.useState(''); const [status, setStatus] = React.useState('idle'); @@ -353,6 +378,7 @@ export const DocsGPTWidget = ({ const [conversationId, setConversationId] = React.useState(null) const [open, setOpen] = React.useState(false) const [eventInterrupt, setEventInterrupt] = React.useState(false); //click or scroll by user while autoScrolling + const isBubbleHovered = useRef(false) const endMessageRef = React.useRef(null); const md = new MarkdownIt(); @@ -376,6 +402,36 @@ export const DocsGPTWidget = ({ !eventInterrupt && scrollToBottom(endMessageRef.current); }, [queries.length, queries[queries.length - 1]?.response]); + async function handleFeedback(feedback: FEEDBACK, index: number) { + let query = queries[index] + if (!query.response) + return; + if (query.feedback != feedback) { + sendFeedback({ + question: query.prompt, + answer: query.response, + feedback: feedback, + apikey: apiKey + }, apiHost) + .then(res => { + if (res.status == 200) { + query.feedback = feedback; + setQueries((prev: Query[]) => { + return prev.map((q, i) => (i === index ? query : q)); + }); + } + }) + + } + else { + delete query.feedback; + setQueries((prev: Query[]) => { + return prev.map((q, i) => (i === index ? query : q)); + }); + + } + } + async function stream(question: string) { setStatus('loading') try { @@ -461,6 +517,8 @@ export const DocsGPTWidget = ({ { queries.length > 0 ? queries?.map((query, index) => { + console.log(query.feedback); + return ( { @@ -473,7 +531,7 @@ export const DocsGPTWidget = ({ } { - query.response ? + query.response ? { isBubbleHovered.current = true }} type='ANSWER'> + + {collectFeedback && + + handleFeedback("LIKE", index)} /> + handleFeedback("DISLIKE", index)} /> + } :
{ @@ -518,7 +596,7 @@ export const DocsGPTWidget = ({ type='text' placeholder="What do you want to do?" /> + disabled={prompt.trim().length == 0 || status !== 'idle'}> diff --git a/extensions/react-widget/src/requests/streamingApi.ts b/extensions/react-widget/src/requests/streamingApi.ts index b594915f..9cb9fddc 100644 --- a/extensions/react-widget/src/requests/streamingApi.ts +++ b/extensions/react-widget/src/requests/streamingApi.ts @@ -1,3 +1,4 @@ +import { FEEDBACK } from "@/types"; interface HistoryItem { prompt: string; response?: string; @@ -11,6 +12,12 @@ interface FetchAnswerStreamingProps { apiHost?: string; onEvent?: (event: MessageEvent) => void; } +interface FeedbackPayload { + question: string; + answer: string; + apikey: string; + feedback: FEEDBACK; +} export function fetchAnswerStreaming({ question = '', apiKey = '', @@ -20,12 +27,12 @@ export function fetchAnswerStreaming({ onEvent = () => { console.log("Event triggered, but no handler provided."); } }: FetchAnswerStreamingProps): Promise { return new Promise((resolve, reject) => { - const body= { + const body = { question: question, history: JSON.stringify(history), conversation_id: conversationId, model: 'default', - api_key:apiKey + api_key: apiKey }; fetch(apiHost + '/stream', { method: 'POST', @@ -80,4 +87,20 @@ export function fetchAnswerStreaming({ reject(error); }); }); -} \ No newline at end of file +} + + +export const sendFeedback = (payload: FeedbackPayload,apiHost:string): Promise => { + return fetch(`${apiHost}/api/feedback`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + question: payload.question, + answer: payload.answer, + feedback: payload.feedback, + api_key:payload.apikey + }), + }); +}; \ No newline at end of file diff --git a/extensions/react-widget/src/types/index.ts b/extensions/react-widget/src/types/index.ts index cb46f06b..a55b6342 100644 --- a/extensions/react-widget/src/types/index.ts +++ b/extensions/react-widget/src/types/index.ts @@ -23,4 +23,5 @@ export interface WidgetProps { theme?:THEME, buttonIcon?:string; buttonBg?:string; + collectFeedback?:boolean } \ No newline at end of file From 3228b883129b292ff9ed3cd6214dd189ccadbc97 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Tue, 10 Sep 2024 02:44:10 +0530 Subject: [PATCH 2/4] widget: minor changes --- .../src/components/DocsGPTWidget.tsx | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/extensions/react-widget/src/components/DocsGPTWidget.tsx b/extensions/react-widget/src/components/DocsGPTWidget.tsx index 7b0f5bb3..cf91b9b1 100644 --- a/extensions/react-widget/src/components/DocsGPTWidget.tsx +++ b/extensions/react-widget/src/components/DocsGPTWidget.tsx @@ -65,13 +65,9 @@ const GlobalStyles = createGlobalStyle` background-color: #646464; color: #fff !important; } -.feedback > .selected{ - fill: #fff; - display: block !important; -} -.feedback > .default{ - fill:#000; - display:block; +.response code { + white-space: pre-wrap !important; + line-break: loose !important; } `; const Overlay = styled.div` @@ -205,15 +201,22 @@ const Conversation = styled.div<{ size: string }>` width:${props => props.size === 'large' ? '90vw' : props.size === 'medium' ? '60vw' : '400px'} !important; } `; - +const Feedback = styled.div` + background-color: transparent; + font-weight: normal; + gap: 12px; + display: flex; + padding: 6px; + clear: both; +`; const MessageBubble = styled.div<{ type: MESSAGE_TYPE }>` display: block; font-size: 16px; position: relative; - width: 100%; + width: 100%;; float: right; margin: 0rem; - &:hover .feedback-icons { + &:hover ${Feedback} * { visibility: visible !important; } `; @@ -331,14 +334,7 @@ const HeroDescription = styled.p` font-size: 14px; line-height: 1.5; `; -const Feedback = styled.div` - background-color: transparent; - font-weight: normal; - gap: 12px; - display: flex; - padding: 6px; - clear: both; -` + const Hero = ({ title, description, theme }: { title: string, description: string, theme: string }) => { return ( <> @@ -359,8 +355,8 @@ const Hero = ({ title, description, theme }: { title: string, description: strin ); }; export const DocsGPTWidget = ({ - apiHost = 'http://127.0.0.1:7091', - apiKey = "220a4876-014d-47c8-996b-936fbefd3a22",//'82962c9a-aa77-4152-94e5-a4f84fd44c6a', + apiHost = 'https://gptcloud.arc53.com', + apiKey = '82962c9a-aa77-4152-94e5-a4f84fd44c6a', avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png', title = 'Get AI assistance', description = 'DocsGPT\'s AI Chatbot is here to help', @@ -370,7 +366,7 @@ export const DocsGPTWidget = ({ theme = 'dark', buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/message.svg', buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)', - collectFeedback = false + collectFeedback = true }: WidgetProps) => { const [prompt, setPrompt] = React.useState(''); const [status, setStatus] = React.useState('idle'); @@ -543,22 +539,20 @@ export const DocsGPTWidget = ({ {collectFeedback && - + handleFeedback("LIKE", index)} /> handleFeedback("DISLIKE", index)} /> } From 61fdcec511337a99eb6958cab67be6251b8ced54 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Tue, 10 Sep 2024 03:27:49 +0530 Subject: [PATCH 3/4] minor change --- extensions/react-widget/src/components/DocsGPTWidget.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/extensions/react-widget/src/components/DocsGPTWidget.tsx b/extensions/react-widget/src/components/DocsGPTWidget.tsx index cf91b9b1..83defbcf 100644 --- a/extensions/react-widget/src/components/DocsGPTWidget.tsx +++ b/extensions/react-widget/src/components/DocsGPTWidget.tsx @@ -417,7 +417,7 @@ export const DocsGPTWidget = ({ }); } }) - + .catch(err => console.log("Connection failed",err)) } else { delete query.feedback; @@ -513,8 +513,6 @@ export const DocsGPTWidget = ({ { queries.length > 0 ? queries?.map((query, index) => { - console.log(query.feedback); - return ( { From e318228a0889b9ae3590926d115dea1887a802be Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Tue, 10 Sep 2024 16:26:13 +0530 Subject: [PATCH 4/4] purge unwanted linting --- application/api/user/routes.py | 169 +++++++++++---------------- extensions/react-widget/package.json | 14 ++- 2 files changed, 81 insertions(+), 102 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index c49b43fd..91e343fc 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -2,8 +2,6 @@ import os import uuid import shutil from flask import Blueprint, request, jsonify -from urllib.parse import urlparse -import requests from pymongo import MongoClient from bson.objectid import ObjectId from bson.binary import Binary, UuidRepresentation @@ -17,7 +15,7 @@ from application.vectorstore.vector_creator import VectorCreator mongo = MongoClient(settings.MONGO_URI) db = mongo["docsgpt"] conversations_collection = db["conversations"] -vectors_collection = db["vectors"] +sources_collection = db["sources"] prompts_collection = db["prompts"] feedback_collection = db["feedback"] api_key_collection = db["api_keys"] @@ -102,7 +100,7 @@ def delete_by_ids(): return {"status": "error"} if settings.VECTOR_STORE == "faiss": - result = vectors_collection.delete_index(ids=ids) + result = sources_collection.delete_index(ids=ids) if result: return {"status": "ok"} return {"status": "error"} @@ -112,26 +110,24 @@ def delete_by_ids(): def delete_old(): """Delete old indexes.""" import shutil - - path = request.args.get("path") - dirs = path.split("/") - dirs_clean = [] - for i in range(0, len(dirs)): - dirs_clean.append(secure_filename(dirs[i])) - # check that path strats with indexes or vectors - - if dirs_clean[0] not in ["indexes", "vectors"]: - return {"status": "error"} - path_clean = "/".join(dirs_clean) - vectors_collection.delete_one({"name": dirs_clean[-1], "user": dirs_clean[-2]}) + source_id = request.args.get("source_id") + doc = sources_collection.find_one({ + "_id": ObjectId(source_id), + "user": "local", + }) + if(doc is None): + return {"status":"not found"},404 if settings.VECTOR_STORE == "faiss": try: - shutil.rmtree(os.path.join(current_dir, path_clean)) + shutil.rmtree(os.path.join(current_dir, str(doc["_id"]))) except FileNotFoundError: pass else: - vetorstore = VectorCreator.create_vectorstore(settings.VECTOR_STORE, path=os.path.join(current_dir, path_clean)) + vetorstore = VectorCreator.create_vectorstore(settings.VECTOR_STORE, source_id=str(doc["_id"])) vetorstore.delete_index() + sources_collection.delete_one({ + "_id": ObjectId(source_id), + }) return {"status": "ok"} @@ -234,52 +230,36 @@ def combined_json(): data = [ { "name": "default", - "language": "default", - "version": "", - "description": "default", - "fullName": "default", "date": "default", - "docLink": "default", "model": settings.EMBEDDINGS_NAME, "location": "remote", "tokens": "", + "retriever": "classic", } ] # structure: name, language, version, description, fullName, date, docLink - # append data from vectors_collection in sorted order in descending order of date - for index in vectors_collection.find({"user": user}).sort("date", -1): + # append data from sources_collection in sorted order in descending order of date + for index in sources_collection.find({"user": user}).sort("date", -1): data.append( { + "id": str(index["_id"]), "name": index["name"], - "language": index["language"], - "version": "", - "description": index["name"], - "fullName": index["name"], "date": index["date"], - "docLink": index["location"], "model": settings.EMBEDDINGS_NAME, "location": "local", "tokens": index["tokens"] if ("tokens" in index.keys()) else "", + "retriever": index["retriever"] if ("retriever" in index.keys()) else "classic", } ) - if settings.VECTOR_STORE == "faiss": - data_remote = requests.get("https://d3dg1063dc54p9.cloudfront.net/combined.json").json() - for index in data_remote: - index["location"] = "remote" - data.append(index) if "duckduck_search" in settings.RETRIEVERS_ENABLED: data.append( { "name": "DuckDuckGo Search", - "language": "en", - "version": "", - "description": "duckduck_search", - "fullName": "DuckDuckGo Search", "date": "duckduck_search", - "docLink": "duckduck_search", "model": settings.EMBEDDINGS_NAME, "location": "custom", "tokens": "", + "retriever": "duckduck_search", } ) if "brave_search" in settings.RETRIEVERS_ENABLED: @@ -287,14 +267,11 @@ def combined_json(): { "name": "Brave Search", "language": "en", - "version": "", - "description": "brave_search", - "fullName": "Brave Search", "date": "brave_search", - "docLink": "brave_search", "model": settings.EMBEDDINGS_NAME, "location": "custom", "tokens": "", + "retriever": "brave_search", } ) @@ -303,39 +280,13 @@ def combined_json(): @user.route("/api/docs_check", methods=["POST"]) def check_docs(): - # check if docs exist in a vectorstore folder data = request.get_json() - # split docs on / and take first part - if data["docs"].split("/")[0] == "local": - return {"status": "exists"} + vectorstore = "vectors/" + secure_filename(data["docs"]) - base_path = "https://raw.githubusercontent.com/arc53/DocsHUB/main/" if os.path.exists(vectorstore) or data["docs"] == "default": return {"status": "exists"} else: - file_url = urlparse(base_path + vectorstore + "index.faiss") - - if ( - file_url.scheme in ["https"] - and file_url.netloc == "raw.githubusercontent.com" - and file_url.path.startswith("/arc53/DocsHUB/main/") - ): - r = requests.get(file_url.geturl()) - if r.status_code != 200: - return {"status": "null"} - else: - if not os.path.exists(vectorstore): - os.makedirs(vectorstore) - with open(vectorstore + "index.faiss", "wb") as f: - f.write(r.content) - - r = requests.get(base_path + vectorstore + "index.pkl") - with open(vectorstore + "index.pkl", "wb") as f: - f.write(r.content) - else: - return {"status": "null"} - - return {"status": "loaded"} + return {"status": "not found"} @user.route("/api/create_prompt", methods=["POST"]) @@ -422,12 +373,23 @@ def get_api_keys(): keys = api_key_collection.find({"user": user}) list_keys = [] for key in keys: + if "source" in key and isinstance(key["source"],DBRef): + source = db.dereference(key["source"]) + if source is None: + continue + else: + source_name = source["name"] + elif "retriever" in key: + source_name = key["retriever"] + else: + continue + list_keys.append( { "id": str(key["_id"]), "name": key["name"], "key": key["key"][:4] + "..." + key["key"][-4:], - "source": key["source"], + "source": source_name, "prompt_id": key["prompt_id"], "chunks": key["chunks"], } @@ -439,21 +401,22 @@ def get_api_keys(): def create_api_key(): data = request.get_json() name = data["name"] - source = data["source"] prompt_id = data["prompt_id"] chunks = data["chunks"] key = str(uuid.uuid4()) user = "local" - resp = api_key_collection.insert_one( - { - "name": name, - "key": key, - "source": source, - "user": user, - "prompt_id": prompt_id, - "chunks": chunks, - } - ) + new_api_key = { + "name": name, + "key": key, + "user": user, + "prompt_id": prompt_id, + "chunks": chunks, + } + if "source" in data and ObjectId.is_valid(data["source"]): + new_api_key["source"] = DBRef("sources", ObjectId(data["source"])) + if "retriever" in data: + new_api_key["retriever"] = data["retriever"] + resp = api_key_collection.insert_one(new_api_key) new_id = str(resp.inserted_id) return {"id": new_id, "key": key} @@ -481,26 +444,31 @@ def share_conversation(): isPromptable = request.args.get("isPromptable").lower() == "true" conversation = conversations_collection.find_one({"_id": ObjectId(conversation_id)}) + if(conversation is None): + raise Exception("Conversation does not exist") current_n_queries = len(conversation["queries"]) ##generate binary representation of uuid explicit_binary = Binary.from_uuid(uuid.uuid4(), UuidRepresentation.STANDARD) if isPromptable: - source = "default" if "source" not in data else data["source"] prompt_id = "default" if "prompt_id" not in data else data["prompt_id"] chunks = "2" if "chunks" not in data else data["chunks"] name = conversation["name"] + "(shared)" - pre_existing_api_document = api_key_collection.find_one( - { + new_api_key_data = { "prompt_id": prompt_id, "chunks": chunks, - "source": source, "user": user, } + if "source" in data and ObjectId.is_valid(data["source"]): + new_api_key_data["source"] = DBRef("sources",ObjectId(data["source"])) + elif "retriever" in data: + new_api_key_data["retriever"] = data["retriever"] + + pre_existing_api_document = api_key_collection.find_one( + new_api_key_data ) - api_uuid = str(uuid.uuid4()) if pre_existing_api_document: api_uuid = pre_existing_api_document["key"] pre_existing = shared_conversations_collections.find_one( @@ -538,17 +506,16 @@ def share_conversation(): ) return jsonify({"success": True, "identifier": str(explicit_binary.as_uuid())}) else: - api_key_collection.insert_one( - { - "name": name, - "key": api_uuid, - "source": source, - "user": user, - "prompt_id": prompt_id, - "chunks": chunks, - } - ) - shared_conversations_collections.insert_one( + + api_uuid = str(uuid.uuid4()) + new_api_key_data["key"] = api_uuid + new_api_key_data["name"] = name + if "source" in data and ObjectId.is_valid(data["source"]): + new_api_key_data["source"] = DBRef("sources", ObjectId(data["source"])) + if "retriever" in data: + new_api_key_data["retriever"] = data["retriever"] + api_key_collection.insert_one(new_api_key_data) + shared_conversations_collections.insert_one( { "uuid": explicit_binary, "conversation_id": { @@ -560,7 +527,7 @@ def share_conversation(): "user": user, "api_key": api_uuid, } - ) + ) ## Identifier as route parameter in frontend return ( jsonify({"success": True, "identifier": str(explicit_binary.as_uuid())}), diff --git a/extensions/react-widget/package.json b/extensions/react-widget/package.json index 813478e2..d449d0a3 100644 --- a/extensions/react-widget/package.json +++ b/extensions/react-widget/package.json @@ -1,5 +1,5 @@ { - "name": "docsgpt-react", + "name": "docsgpt", "version": "0.4.2", "private": false, "description": "DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.", @@ -11,6 +11,18 @@ "dist", "package.json" ], + "targets": { + "modern": { + "engines": { + "browsers": "Chrome 80" + } + }, + "legacy": { + "engines": { + "browsers": "> 0.5%, last 2 versions, not dead" + } + } + }, "@parcel/resolver-default": { "packageExports": true },