Merge pull request #926 from siiddhantt/feature

Feature: Logging token usage info to MongoDB

.gitignore

@@ -75,6 +75,7 @@ target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
+**/*.ipynb
 
 # IPython
 profile_default/

SECURITY.md

@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+Supported Versions:
+
+Currently, we support security patches by committing changes and bumping the version published on Github.
+
+## Reporting a Vulnerability
+
+Found a vulnerability? Please email us:
+
+security@arc53.com
+

application/api/answer/routes.py

@@ -8,17 +8,14 @@ import traceback
 
 from pymongo import MongoClient
 from bson.objectid import ObjectId
-from transformers import GPT2TokenizerFast
-
 
 
 from application.core.settings import settings
-from application.vectorstore.vector_creator import VectorCreator
 from application.llm.llm_creator import LLMCreator
+from application.retriever.retriever_creator import RetrieverCreator
 from application.error import bad_request
 
 
-
 logger = logging.getLogger(__name__)
 
 mongo = MongoClient(settings.MONGO_URI)
@@ -26,17 +23,23 @@ db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
 vectors_collection = db["vectors"]
 prompts_collection = db["prompts"]
-answer = Blueprint('answer', __name__)
+api_key_collection = db["api_keys"]
+answer = Blueprint("answer", __name__)
 
-if settings.LLM_NAME == "gpt4":
-    gpt_model = 'gpt-4'
+gpt_model = ""
+# to have some kind of default behaviour
+if settings.LLM_NAME == "openai":
+    gpt_model = "gpt-3.5-turbo"
 elif settings.LLM_NAME == "anthropic":
-    gpt_model = 'claude-2'
-else:
-    gpt_model = 'gpt-3.5-turbo'
+    gpt_model = "claude-2"
+
+if settings.MODEL_NAME:  # in case there is particular model name configured
+    gpt_model = settings.MODEL_NAME
 
 # load the prompts
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
 with open(os.path.join(current_dir, "prompts", "chat_combine_default.txt"), "r") as f:
     chat_combine_template = f.read()
 
@@ -47,7 +50,7 @@ with open(os.path.join(current_dir, "prompts", "chat_combine_creative.txt"), "r"
     chat_combine_creative = f.read()
 
 with open(os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r") as f:
-    chat_combine_strict = f.read()    
+    chat_combine_strict = f.read()
 
 api_key_set = settings.API_KEY is not None
 embeddings_key_set = settings.EMBEDDINGS_KEY is not None
@@ -58,11 +61,6 @@ async def async_generate(chain, question, chat_history):
     return result
 
 
-def count_tokens(string):
-    tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
-    return len(tokenizer(string)['input_ids'])
-
-
 def run_async_chain(chain, question, chat_history):
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
@@ -75,10 +73,17 @@ def run_async_chain(chain, question, chat_history):
     return result
 
 
+def get_data_from_api_key(api_key):
+    data = api_key_collection.find_one({"key": api_key})
+    if data is None:
+        return bad_request(401, "Invalid API key")
+    return data
+
+
 def get_vectorstore(data):
     if "active_docs" in data:
         if data["active_docs"].split("/")[0] == "default":
-                vectorstore = ""
+            vectorstore = ""
         elif data["active_docs"].split("/")[0] == "local":
             vectorstore = "indexes/" + data["active_docs"]
         else:
@@ -92,83 +97,99 @@ def get_vectorstore(data):
 
 
 def is_azure_configured():
-    return settings.OPENAI_API_BASE and settings.OPENAI_API_VERSION and settings.AZURE_DEPLOYMENT_NAME
+    return (
+        settings.OPENAI_API_BASE
+        and settings.OPENAI_API_VERSION
+        and settings.AZURE_DEPLOYMENT_NAME
+    )
 
 
-def complete_stream(question, docsearch, chat_history, api_key, prompt_id, conversation_id):
-    llm = LLMCreator.create_llm(settings.LLM_NAME, api_key=api_key)
-
-    if prompt_id == 'default':
-        prompt = chat_combine_template
-    elif prompt_id == 'creative':
-        prompt = chat_combine_creative
-    elif prompt_id == 'strict':
-        prompt = chat_combine_strict
-    else:
-        prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})["content"]
-
-    docs = docsearch.search(question, k=2)
-    if settings.LLM_NAME == "llama.cpp":
-        docs = [docs[0]]
-    # join all page_content together with a newline
-    docs_together = "\n".join([doc.page_content for doc in docs])
-    p_chat_combine = prompt.replace("{summaries}", docs_together)
-    messages_combine = [{"role": "system", "content": p_chat_combine}]
-    source_log_docs = []
-    for doc in docs:
-        if doc.metadata:
-            source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content})
-        else:
-            source_log_docs.append({"title": doc.page_content, "text": doc.page_content})
-
-    if len(chat_history) > 1:
-        tokens_current_history = 0
-        # count tokens in history
-        chat_history.reverse()
-        for i in chat_history:
-            if "prompt" in i and "response" in i:
-                tokens_batch = count_tokens(i["prompt"]) + count_tokens(i["response"])
-                if tokens_current_history + tokens_batch < settings.TOKENS_MAX_HISTORY:
-                    tokens_current_history += tokens_batch
-                    messages_combine.append({"role": "user", "content": i["prompt"]})
-                    messages_combine.append({"role": "system", "content": i["response"]})
-    messages_combine.append({"role": "user", "content": question})
-
-    response_full = ""
-    completion = llm.gen_stream(model=gpt_model, engine=settings.AZURE_DEPLOYMENT_NAME,
-                                messages=messages_combine)
-    for line in completion:
-        data = json.dumps({"answer": str(line)})
-        response_full += str(line)
-        yield f"data: {data}\n\n"
-
-    # save conversation to database
-    if conversation_id is not None:
+def save_conversation(conversation_id, question, response, source_log_docs, llm):
+    if conversation_id is not None and conversation_id != "None":
         conversations_collection.update_one(
             {"_id": ObjectId(conversation_id)},
-            {"$push": {"queries": {"prompt": question, "response": response_full, "sources": source_log_docs}}},
+            {
+                "$push": {
+                    "queries": {
+                        "prompt": question,
+                        "response": response,
+                        "sources": source_log_docs,
+                    }
+                }
+            },
         )
 
     else:
         # create new conversation
         # generate summary
-        messages_summary = [{"role": "assistant", "content": "Summarise following conversation in no more than 3 "
-                                                             "words, respond ONLY with the summary, use the same "
-                                                             "language as the system \n\nUser: " + question + "\n\n" +
-                                                             "AI: " +
-                                                             response_full},
-                            {"role": "user", "content": "Summarise following conversation in no more than 3 words, "
-                                                        "respond ONLY with the summary, use the same language as the "
-                                                        "system"}]
+        messages_summary = [
+            {
+                "role": "assistant",
+                "content": "Summarise following conversation in no more than 3 "
+                "words, respond ONLY with the summary, use the same "
+                "language as the system \n\nUser: "
+                + question
+                + "\n\n"
+                + "AI: "
+                + response,
+            },
+            {
+                "role": "user",
+                "content": "Summarise following conversation in no more than 3 words, "
+                "respond ONLY with the summary, use the same language as the "
+                "system",
+            },
+        ]
 
-        completion = llm.gen(model=gpt_model, engine=settings.AZURE_DEPLOYMENT_NAME,
-                             messages=messages_summary, max_tokens=30)
+        completion = llm.gen(model=gpt_model, messages=messages_summary, max_tokens=30)
         conversation_id = conversations_collection.insert_one(
-            {"user": "local",
-             "date": datetime.datetime.utcnow(),
-             "name": completion,
-             "queries": [{"prompt": question, "response": response_full, "sources": source_log_docs}]}
+            {
+                "user": "local",
+                "date": datetime.datetime.utcnow(),
+                "name": completion,
+                "queries": [
+                    {
+                        "prompt": question,
+                        "response": response,
+                        "sources": source_log_docs,
+                    }
+                ],
+            }
         ).inserted_id
+    return conversation_id
+
+
+def get_prompt(prompt_id):
+    if prompt_id == "default":
+        prompt = chat_combine_template
+    elif prompt_id == "creative":
+        prompt = chat_combine_creative
+    elif prompt_id == "strict":
+        prompt = chat_combine_strict
+    else:
+        prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})["content"]
+    return prompt
+
+
+def complete_stream(question, retriever, conversation_id, user_api_key):
+
+    response_full = ""
+    source_log_docs = []
+    answer = retriever.gen()
+    for line in answer:
+        if "answer" in line:
+            response_full += str(line["answer"])
+            data = json.dumps(line)
+            yield f"data: {data}\n\n"
+        elif "source" in line:
+            source_log_docs.append(line["source"])
+
+    llm = LLMCreator.create_llm(
+        settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
+    )
+    conversation_id = save_conversation(
+        conversation_id, question, response_full, source_log_docs, llm
+    )
 
     # send data.type = "end" to indicate that the stream has ended as json
     data = json.dumps({"type": "id", "id": str(conversation_id)})
@@ -182,36 +203,68 @@ def stream():
     data = request.get_json()
     # get parameter from url question
     question = data["question"]
-    history = data["history"]
-    # history to json object from string
-    history = json.loads(history)
-    conversation_id = data["conversation_id"]
-    if 'prompt_id' in data:
+    if "history" not in data:
+        history = []
+    else:
+        history = data["history"]
+        history = json.loads(history)
+    if "conversation_id" not in data:
+        conversation_id = None
+    else:
+        conversation_id = data["conversation_id"]
+    if "prompt_id" in data:
         prompt_id = data["prompt_id"]
     else:
-        prompt_id = 'default'
+        prompt_id = "default"
+    if "selectedDocs" in data and data["selectedDocs"] is None:
+        chunks = 0
+    elif "chunks" in data:
+        chunks = int(data["chunks"])
+    else:
+        chunks = 2
+
+    prompt = get_prompt(prompt_id)
 
     # check if active_docs is set
 
-    if not api_key_set:
-        api_key = data["api_key"]
+    if "api_key" in data:
+        data_key = get_data_from_api_key(data["api_key"])
+        source = {"active_docs": data_key["source"]}
+        user_api_key = data["api_key"]
+    elif "active_docs" in data:
+        source = {"active_docs": data["active_docs"]}
+        user_api_key = None
     else:
-        api_key = settings.API_KEY
-    if not embeddings_key_set:
-        embeddings_key = data["embeddings_key"]
+        source = {}
+        user_api_key = None
+
+    if (
+        source["active_docs"].split("/")[0] == "default"
+        or source["active_docs"].split("/")[0] == "local"
+    ):
+        retriever_name = "classic"
     else:
-        embeddings_key = settings.EMBEDDINGS_KEY
-    if "active_docs" in data:
-        vectorstore = get_vectorstore({"active_docs": data["active_docs"]})
-    else:
-        vectorstore = ""
-    docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key)
+        retriever_name = source["active_docs"]
+
+    retriever = RetrieverCreator.create_retriever(
+        retriever_name,
+        question=question,
+        source=source,
+        chat_history=history,
+        prompt=prompt,
+        chunks=chunks,
+        gpt_model=gpt_model,
+        user_api_key=user_api_key,
+    )
 
     return Response(
-        complete_stream(question, docsearch,
-                        chat_history=history, api_key=api_key,
-                        prompt_id=prompt_id,
-                        conversation_id=conversation_id), mimetype="text/event-stream"
+        complete_stream(
+            question=question,
+            retriever=retriever,
+            conversation_id=conversation_id,
+            user_api_key=user_api_key,
+        ),
+        mimetype="text/event-stream",
     )
 
 
@@ -219,121 +272,72 @@ def stream():
 def api_answer():
     data = request.get_json()
     question = data["question"]
-    history = data["history"]
+    if "history" not in data:
+        history = []
+    else:
+        history = data["history"]
     if "conversation_id" not in data:
         conversation_id = None
     else:
         conversation_id = data["conversation_id"]
     print("-" * 5)
-    if not api_key_set:
-        api_key = data["api_key"]
-    else:
-        api_key = settings.API_KEY
-    if not embeddings_key_set:
-        embeddings_key = data["embeddings_key"]
-    else:
-        embeddings_key = settings.EMBEDDINGS_KEY
-    if 'prompt_id' in data:
+    if "prompt_id" in data:
         prompt_id = data["prompt_id"]
     else:
-        prompt_id = 'default'
-
-    if prompt_id == 'default':
-        prompt = chat_combine_template
-    elif prompt_id == 'creative':
-        prompt = chat_combine_creative
-    elif prompt_id == 'strict':
-        prompt = chat_combine_strict
+        prompt_id = "default"
+    if "chunks" in data:
+        chunks = int(data["chunks"])
     else:
-        prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})["content"]
+        chunks = 2
+
+    prompt = get_prompt(prompt_id)
 
     # use try and except  to check for exception
     try:
         # check if the vectorstore is set
-        vectorstore = get_vectorstore(data)
-        # loading the index and the store and the prompt template
-        # Note if you have used other embeddings than OpenAI, you need to change the embeddings
-        docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key)
-
-
-        llm = LLMCreator.create_llm(settings.LLM_NAME, api_key=api_key)
-
-
-
-        docs = docsearch.search(question, k=2)
-        # join all page_content together with a newline
-        docs_together = "\n".join([doc.page_content for doc in docs])
-        p_chat_combine = prompt.replace("{summaries}", docs_together)
-        messages_combine = [{"role": "system", "content": p_chat_combine}]
-        source_log_docs = []
-        for doc in docs:
-            if doc.metadata:
-                source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content})
-            else:
-                source_log_docs.append({"title": doc.page_content, "text": doc.page_content})
-        # join all page_content together with a newline
-
-
-        if len(history) > 1:
-            tokens_current_history = 0
-            # count tokens in history
-            history.reverse()
-            for i in history:
-                if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(i["response"])
-                    if tokens_current_history + tokens_batch < settings.TOKENS_MAX_HISTORY:
-                        tokens_current_history += tokens_batch
-                        messages_combine.append({"role": "user", "content": i["prompt"]})
-                        messages_combine.append({"role": "system", "content": i["response"]})
-        messages_combine.append({"role": "user", "content": question})
-
-
-        completion = llm.gen(model=gpt_model, engine=settings.AZURE_DEPLOYMENT_NAME,
-                                    messages=messages_combine)
-
-
-        result = {"answer": completion, "sources": source_log_docs}
-        logger.debug(result)
-
-        # generate conversationId
-        if conversation_id is not None:
-            conversations_collection.update_one(
-                {"_id": ObjectId(conversation_id)},
-                {"$push": {"queries": {"prompt": question,
-                                       "response": result["answer"], "sources": result['sources']}}},
-            )
-
+        if "api_key" in data:
+            data_key = get_data_from_api_key(data["api_key"])
+            source = {"active_docs": data_key["source"]}
+            user_api_key = data["api_key"]
         else:
-            # create new conversation
-            # generate summary
-            messages_summary = [
-                {"role": "assistant", "content": "Summarise following conversation in no more than 3 words, "
-                    "respond ONLY with the summary, use the same language as the system \n\n"
-                    "User: " + question + "\n\n" + "AI: " + result["answer"]},
-                {"role": "user", "content": "Summarise following conversation in no more than 3 words, "
-                    "respond ONLY with the summary, use the same language as the system"}
-            ]
+            source = {data}
+            user_api_key = None
 
-            completion = llm.gen(
-                model=gpt_model,
-                engine=settings.AZURE_DEPLOYMENT_NAME,
-                messages=messages_summary,
-                max_tokens=30
-            )
-            conversation_id = conversations_collection.insert_one(
-                {"user": "local",
-                "date": datetime.datetime.utcnow(),
-                "name": completion,
-                "queries": [{"prompt": question, "response": result["answer"], "sources": source_log_docs}]}
-            ).inserted_id
+        if (
+            source["active_docs"].split("/")[0] == "default"
+            or source["active_docs"].split("/")[0] == "local"
+        ):
+            retriever_name = "classic"
+        else:
+            retriever_name = source["active_docs"]
 
-        result["conversation_id"] = str(conversation_id)
+        retriever = RetrieverCreator.create_retriever(
+            retriever_name,
+            question=question,
+            source=source,
+            chat_history=history,
+            prompt=prompt,
+            chunks=chunks,
+            gpt_model=gpt_model,
+            user_api_key=user_api_key,
+        )
+        source_log_docs = []
+        response_full = ""
+        for line in retriever.gen():
+            if "source" in line:
+                source_log_docs.append(line["source"])
+            elif "answer" in line:
+                response_full += line["answer"]
+
+        llm = LLMCreator.create_llm(
+            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
+        )
+
+        result = {"answer": response_full, "sources": source_log_docs}
+        result["conversation_id"] = save_conversation(
+            conversation_id, question, response_full, source_log_docs, llm
+        )
 
-        # mock result
-        # result = {
-        #     "answer": "The answer is 42",
-        #     "sources": ["https://en.wikipedia.org/wiki/42_(number)", "https://en.wikipedia.org/wiki/42_(number)"]
-        # }
         return result
     except Exception as e:
         # print whole traceback
@@ -348,27 +352,38 @@ def api_search():
     # get parameter from url question
     question = data["question"]
 
-    if not embeddings_key_set:
-        if "embeddings_key" in data:
-            embeddings_key = data["embeddings_key"]
-        else:
-            embeddings_key = settings.EMBEDDINGS_KEY
+    if "api_key" in data:
+        data_key = get_data_from_api_key(data["api_key"])
+        source = {"active_docs": data_key["source"]}
+        user_api_key = data["api_key"]
+    elif "active_docs" in data:
+        source = {"active_docs": data["active_docs"]}
+        user_api_key = None
     else:
-        embeddings_key = settings.EMBEDDINGS_KEY
-    if "active_docs" in data:
-        vectorstore = get_vectorstore({"active_docs": data["active_docs"]})
+        source = {}
+        user_api_key = None
+    if "chunks" in data:
+        chunks = int(data["chunks"])
     else:
-        vectorstore = ""
-    docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key)
+        chunks = 2
 
-    docs = docsearch.search(question, k=2)
-
-    source_log_docs = []
-    for doc in docs:
-        if doc.metadata:
-            source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content})
-        else:
-            source_log_docs.append({"title": doc.page_content, "text": doc.page_content})
-        #yield f"data:{data}\n\n"
-    return source_log_docs
+    if (
+        source["active_docs"].split("/")[0] == "default"
+        or source["active_docs"].split("/")[0] == "local"
+    ):
+        retriever_name = "classic"
+    else:
+        retriever_name = source["active_docs"]
 
+    retriever = RetrieverCreator.create_retriever(
+        retriever_name,
+        question=question,
+        source=source,
+        chat_history=[],
+        prompt="default",
+        chunks=chunks,
+        gpt_model=gpt_model,
+        user_api_key=user_api_key,
+    )
+    docs = retriever.search()
+    return docs

application/api/user/routes.py

@@ -1,5 +1,8 @@
 import os
+import uuid
+import shutil
 from flask import Blueprint, request, jsonify
+from urllib.parse import urlparse
 import requests
 from pymongo import MongoClient
 from bson.objectid import ObjectId
@@ -16,6 +19,7 @@ conversations_collection = db["conversations"]
 vectors_collection = db["vectors"]
 prompts_collection = db["prompts"]
 feedback_collection = db["feedback"]
+api_key_collection = db["api_keys"]
 user = Blueprint('user', __name__)
 
 current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -133,30 +137,43 @@ def upload_file():
         return {"status": "no name"}
     job_name = secure_filename(request.form["name"])
     # check if the post request has the file part
-    if "file" not in request.files:
-        print("No file part")
-        return {"status": "no file"}
-    file = request.files["file"]
-    if file.filename == "":
+    files = request.files.getlist("file")
+        
+    if not files or all(file.filename == '' for file in files):
         return {"status": "no file name"}
 
-    if file:
-        filename = secure_filename(file.filename)
-        # save dir
-        save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name)
-        # create dir if not exists
-        if not os.path.exists(save_dir):
-            os.makedirs(save_dir)
-
-        file.save(os.path.join(save_dir, filename))
-        task = ingest.delay(settings.UPLOAD_FOLDER, [".rst", ".md", ".pdf", ".txt", ".docx", 
-        ".csv", ".epub", ".html", ".mdx"],
-         job_name, filename, user)
-        # task id
-        task_id = task.id
-        return {"status": "ok", "task_id": task_id}
+    # Directory where files will be saved
+    save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name)
+    os.makedirs(save_dir, exist_ok=True)
+    
+    if len(files) > 1:
+        # Multiple files; prepare them for zip
+        temp_dir = os.path.join(save_dir, "temp")
+        os.makedirs(temp_dir, exist_ok=True)
+        
+        for file in files:
+            filename = secure_filename(file.filename)
+            file.save(os.path.join(temp_dir, filename))
+        
+        # Use shutil.make_archive to zip the temp directory
+        zip_path = shutil.make_archive(base_name=os.path.join(save_dir, job_name), format='zip', root_dir=temp_dir)
+        final_filename = os.path.basename(zip_path)
+        
+        # Clean up the temporary directory after zipping
+        shutil.rmtree(temp_dir)
     else:
-        return {"status": "error"}
+        # Single file
+        file = files[0]
+        final_filename = secure_filename(file.filename)
+        file_path = os.path.join(save_dir, final_filename)
+        file.save(file_path)
+    
+    # Call ingest with the single file or zipped file
+    task = ingest.delay(settings.UPLOAD_FOLDER, [".rst", ".md", ".pdf", ".txt", ".docx", 
+    ".csv", ".epub", ".html", ".mdx"],
+    job_name, final_filename, user)
+    
+    return {"status": "ok", "task_id": task.id}
     
 @user.route("/api/remote", methods=["POST"])
 def upload_remote():
@@ -234,6 +251,34 @@ def combined_json():
         for index in data_remote:
             index["location"] = "remote"
             data.append(index)
+    if 'duckduck_search' in settings.RETRIEVERS_ENABLED:
+        data.append(
+            {
+                "name": "DuckDuckGo Search",
+                "language": "en",
+                "version": "",
+                "description": "duckduck_search",
+                "fullName": "DuckDuckGo Search",
+                "date": "duckduck_search",
+                "docLink": "duckduck_search",
+                "model": settings.EMBEDDINGS_NAME,
+                "location": "custom",
+            }
+        )
+    if 'brave_search' in settings.RETRIEVERS_ENABLED:
+        data.append(
+            {
+                "name": "Brave Search",
+                "language": "en",
+                "version": "",
+                "description": "brave_search",
+                "fullName": "Brave Search",
+                "date": "brave_search",
+                "docLink": "brave_search",
+                "model": settings.EMBEDDINGS_NAME,
+                "location": "custom",
+            }
+        )
 
     return jsonify(data)
 
@@ -245,25 +290,32 @@ def check_docs():
     # split docs on / and take first part
     if data["docs"].split("/")[0] == "local":
         return {"status": "exists"}
-    vectorstore = "vectors/" + data["docs"]
+    vectorstore = "vectors/" + secure_filename(data["docs"])
     base_path = "https://raw.githubusercontent.com/arc53/DocsHUB/main/"
     if os.path.exists(vectorstore) or data["docs"] == "default":
         return {"status": "exists"}
     else:
-        r = requests.get(base_path + vectorstore + "index.faiss")
+        file_url = urlparse(base_path + vectorstore + "index.faiss")
+        
+        if (
+            file_url.scheme in ['https'] and 
+            file_url.netloc == 'raw.githubusercontent.com' and 
+            file_url.path.startswith('/arc53/DocsHUB/main/')
+        ):
+            r = requests.get(file_url.geturl())
+            if r.status_code != 200:
+                return {"status": "null"}
+            else:
+                if not os.path.exists(vectorstore):
+                    os.makedirs(vectorstore)
+                with open(vectorstore + "index.faiss", "wb") as f:
+                    f.write(r.content)
 
-        if r.status_code != 200:
-            return {"status": "null"}
+                r = requests.get(base_path + vectorstore + "index.pkl")
+                with open(vectorstore + "index.pkl", "wb") as f:
+                    f.write(r.content)
         else:
-            if not os.path.exists(vectorstore):
-                os.makedirs(vectorstore)
-            with open(vectorstore + "index.faiss", "wb") as f:
-                f.write(r.content)
-
-            # download the store
-            r = requests.get(base_path + vectorstore + "index.pkl")
-            with open(vectorstore + "index.pkl", "wb") as f:
-                f.write(r.content)
+            return {"status": "null"}
 
         return {"status": "loaded"}
 
@@ -343,5 +395,52 @@ def update_prompt_name():
 
 
 
+@user.route("/api/get_api_keys", methods=["GET"])
+def get_api_keys():
+    user = "local"
+    keys = api_key_collection.find({"user": user})
+    list_keys = []
+    for key in keys:
+        list_keys.append({
+            "id": str(key["_id"]),
+            "name": key["name"],
+            "key": key["key"][:4] + "..." + key["key"][-4:],
+            "source": key["source"],
+            "prompt_id": key["prompt_id"],
+            "chunks": key["chunks"]
+        })
+    return jsonify(list_keys)
 
+@user.route("/api/create_api_key", methods=["POST"])
+def create_api_key():
+    data = request.get_json()
+    name = data["name"]
+    source = data["source"]
+    prompt_id = data["prompt_id"]
+    chunks = data["chunks"]
+    key = str(uuid.uuid4())
+    user = "local"
+    resp = api_key_collection.insert_one(
+        {
+            "name": name,
+            "key": key,
+            "source": source,
+            "user": user,
+            "prompt_id": prompt_id,
+            "chunks": chunks
+        }
+    )
+    new_id = str(resp.inserted_id)
+    return {"id": new_id, "key": key}
+
+@user.route("/api/delete_api_key", methods=["POST"])
+def delete_api_key():
+    data = request.get_json()
+    id = data["id"]
+    api_key_collection.delete_one(
+        {
+            "_id": ObjectId(id),
+        }
+    )
+    return {"status": "ok"}
 

application/app.py

@@ -40,5 +40,5 @@ def after_request(response):
     return response
 
 if __name__ == "__main__":
-    app.run(debug=True, port=7091)
+    app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)
 

application/core/settings.py

@@ -9,6 +9,7 @@ current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__
 
 class Settings(BaseSettings):
     LLM_NAME: str = "docsgpt"
+    MODEL_NAME: Optional[str] = None # if LLM_NAME is openai, MODEL_NAME can be gpt-4 or gpt-3.5-turbo
     EMBEDDINGS_NAME: str = "huggingface_sentence-transformers/all-mpnet-base-v2"
     CELERY_BROKER_URL: str = "redis://localhost:6379/0"
     CELERY_RESULT_BACKEND: str = "redis://localhost:6379/1"
@@ -17,6 +18,7 @@ class Settings(BaseSettings):
     TOKENS_MAX_HISTORY: int = 150
     UPLOAD_FOLDER: str = "inputs"
     VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant"
+    RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
 
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
 
@@ -58,6 +60,10 @@ class Settings(BaseSettings):
     QDRANT_PATH: Optional[str] = None
     QDRANT_DISTANCE_FUNC: str = "Cosine"
 
+    BRAVE_SEARCH_API_KEY: Optional[str] = None
+
+    FLASK_DEBUG_MODE: bool = False
+
 
 path = Path(__file__).parent.parent.absolute()
 settings = Settings(_env_file=path.joinpath(".env"), _env_file_encoding="utf-8")

application/llm/anthropic.py

@@ -1,21 +1,29 @@
 from application.llm.base import BaseLLM
 from application.core.settings import settings
 
+
 class AnthropicLLM(BaseLLM):
 
-    def __init__(self, api_key=None):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
         from anthropic import Anthropic, HUMAN_PROMPT, AI_PROMPT
-        self.api_key = api_key or settings.ANTHROPIC_API_KEY  # If not provided, use a default from settings
+
+        super().__init__(*args, **kwargs)
+        self.api_key = (
+            api_key or settings.ANTHROPIC_API_KEY
+        )  # If not provided, use a default from settings
+        self.user_api_key = user_api_key
         self.anthropic = Anthropic(api_key=self.api_key)
         self.HUMAN_PROMPT = HUMAN_PROMPT
         self.AI_PROMPT = AI_PROMPT
 
-    def gen(self, model, messages, engine=None, max_tokens=300, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(
+        self, baseself, model, messages, stream=False, max_tokens=300, **kwargs
+    ):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Context \n {context} \n ### Question \n {user_question}"
         if stream:
-            return self.gen_stream(model, prompt, max_tokens, **kwargs)
+            return self.gen_stream(model, prompt, stream, max_tokens, **kwargs)
 
         completion = self.anthropic.completions.create(
             model=model,
@@ -25,9 +33,11 @@ class AnthropicLLM(BaseLLM):
         )
         return completion.completion
 
-    def gen_stream(self, model, messages, engine=None, max_tokens=300, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen_stream(
+        self, baseself, model, messages, stream=True, max_tokens=300, **kwargs
+    ):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Context \n {context} \n ### Question \n {user_question}"
         stream_response = self.anthropic.completions.create(
             model=model,
@@ -37,4 +47,4 @@ class AnthropicLLM(BaseLLM):
         )
 
         for completion in stream_response:
-            yield completion.completion
+            yield completion.completion

application/llm/base.py

@@ -1,14 +1,28 @@
 from abc import ABC, abstractmethod
+from application.usage import gen_token_usage, stream_token_usage
 
 
 class BaseLLM(ABC):
     def __init__(self):
-        pass
+        self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
+
+    def _apply_decorator(self, method, decorator, *args, **kwargs):
+        return decorator(method, *args, **kwargs)
 
     @abstractmethod
-    def gen(self, *args, **kwargs):
+    def _raw_gen(self, model, messages, stream, *args, **kwargs):
         pass
 
+    def gen(self, model, messages, stream=False, *args, **kwargs):
+        return self._apply_decorator(self._raw_gen, gen_token_usage)(
+            self, model=model, messages=messages, stream=stream, *args, **kwargs
+        )
+
     @abstractmethod
-    def gen_stream(self, *args, **kwargs):
+    def _raw_gen_stream(self, model, messages, stream, *args, **kwargs):
         pass
+
+    def gen_stream(self, model, messages, stream=True, *args, **kwargs):
+        return self._apply_decorator(self._raw_gen_stream, stream_token_usage)(
+            self, model=model, messages=messages, stream=stream, *args, **kwargs
+        )

application/llm/docsgpt_provider.py

@@ -2,48 +2,43 @@ from application.llm.base import BaseLLM
 import json
 import requests
 
+
 class DocsGPTAPILLM(BaseLLM):
 
-    def __init__(self, *args, **kwargs):
-        self.endpoint =  "https://llm.docsgpt.co.uk"
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.endpoint = "https://llm.docsgpt.co.uk"
 
-
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(self, baseself, model, messages, stream=False, *args, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
 
         response = requests.post(
-            f"{self.endpoint}/answer",
-            json={
-                "prompt": prompt,
-                "max_new_tokens": 30
-            }
+            f"{self.endpoint}/answer", json={"prompt": prompt, "max_new_tokens": 30}
         )
-        response_clean = response.json()['a'].replace("###", "")
+        response_clean = response.json()["a"].replace("###", "")
 
         return response_clean
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, *args, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
 
         # send prompt to endpoint /stream
         response = requests.post(
             f"{self.endpoint}/stream",
-            json={
-                "prompt": prompt,
-                "max_new_tokens": 256
-            },
-            stream=True
+            json={"prompt": prompt, "max_new_tokens": 256},
+            stream=True,
         )
-    
+
         for line in response.iter_lines():
             if line:
-                #data = json.loads(line)
-                data_str = line.decode('utf-8')
+                # data = json.loads(line)
+                data_str = line.decode("utf-8")
                 if data_str.startswith("data: "):
                     data = json.loads(data_str[6:])
-                    yield data['a']
-                    
+                    yield data["a"]

application/llm/huggingface.py

@@ -1,44 +1,68 @@
 from application.llm.base import BaseLLM
 
+
 class HuggingFaceLLM(BaseLLM):
 
-    def __init__(self, api_key, llm_name='Arc53/DocsGPT-7B',q=False):
+    def __init__(
+        self,
+        api_key=None,
+        user_api_key=None,
+        llm_name="Arc53/DocsGPT-7B",
+        q=False,
+        *args,
+        **kwargs,
+    ):
         global hf
-        
+
         from langchain.llms import HuggingFacePipeline
+
         if q:
             import torch
-            from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig
+            from transformers import (
+                AutoModelForCausalLM,
+                AutoTokenizer,
+                pipeline,
+                BitsAndBytesConfig,
+            )
+
             tokenizer = AutoTokenizer.from_pretrained(llm_name)
             bnb_config = BitsAndBytesConfig(
-                            load_in_4bit=True,
-                            bnb_4bit_use_double_quant=True,
-                            bnb_4bit_quant_type="nf4",
-                            bnb_4bit_compute_dtype=torch.bfloat16
-                        )
-            model = AutoModelForCausalLM.from_pretrained(llm_name,quantization_config=bnb_config)
+                load_in_4bit=True,
+                bnb_4bit_use_double_quant=True,
+                bnb_4bit_quant_type="nf4",
+                bnb_4bit_compute_dtype=torch.bfloat16,
+            )
+            model = AutoModelForCausalLM.from_pretrained(
+                llm_name, quantization_config=bnb_config
+            )
         else:
             from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
             tokenizer = AutoTokenizer.from_pretrained(llm_name)
             model = AutoModelForCausalLM.from_pretrained(llm_name)
-        
+
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
         pipe = pipeline(
-            "text-generation", model=model,
-            tokenizer=tokenizer, max_new_tokens=2000,
-            device_map="auto", eos_token_id=tokenizer.eos_token_id
+            "text-generation",
+            model=model,
+            tokenizer=tokenizer,
+            max_new_tokens=2000,
+            device_map="auto",
+            eos_token_id=tokenizer.eos_token_id,
         )
         hf = HuggingFacePipeline(pipeline=pipe)
 
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
 
         result = hf(prompt)
 
         return result.content
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
 
         raise NotImplementedError("HuggingFaceLLM Streaming is not implemented yet.")
-

application/llm/llama_cpp.py

@@ -1,32 +1,45 @@
 from application.llm.base import BaseLLM
 from application.core.settings import settings
 
+
 class LlamaCpp(BaseLLM):
 
-    def __init__(self, api_key, llm_name=settings.MODEL_PATH, **kwargs):
+    def __init__(
+        self,
+        api_key=None,
+        user_api_key=None,
+        llm_name=settings.MODEL_PATH,
+        *args,
+        **kwargs,
+    ):
         global llama
         try:
             from llama_cpp import Llama
         except ImportError:
-            raise ImportError("Please install llama_cpp using pip install llama-cpp-python")
+            raise ImportError(
+                "Please install llama_cpp using pip install llama-cpp-python"
+            )
 
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
         llama = Llama(model_path=llm_name, n_ctx=2048)
 
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
 
         result = llama(prompt, max_tokens=150, echo=False)
 
         # import sys
         # print(result['choices'][0]['text'].split('### Answer \n')[-1], file=sys.stderr)
-        
-        return result['choices'][0]['text'].split('### Answer \n')[-1]
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+        return result["choices"][0]["text"].split("### Answer \n")[-1]
+
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
 
         result = llama(prompt, max_tokens=150, echo=False, stream=stream)
@@ -35,5 +48,5 @@ class LlamaCpp(BaseLLM):
         # print(list(result), file=sys.stderr)
 
         for item in result:
-            for choice in item['choices']:
-                yield choice['text']
+            for choice in item["choices"]:
+                yield choice["text"]

application/llm/llm_creator.py

@@ -7,22 +7,21 @@ from application.llm.docsgpt_provider import DocsGPTAPILLM
 from application.llm.premai import PremAILLM
 
 
-
 class LLMCreator:
     llms = {
-        'openai': OpenAILLM,
-        'azure_openai': AzureOpenAILLM,
-        'sagemaker': SagemakerAPILLM,
-        'huggingface': HuggingFaceLLM,
-        'llama.cpp': LlamaCpp,
-        'anthropic': AnthropicLLM,
-        'docsgpt': DocsGPTAPILLM,
-        'premai': PremAILLM,
+        "openai": OpenAILLM,
+        "azure_openai": AzureOpenAILLM,
+        "sagemaker": SagemakerAPILLM,
+        "huggingface": HuggingFaceLLM,
+        "llama.cpp": LlamaCpp,
+        "anthropic": AnthropicLLM,
+        "docsgpt": DocsGPTAPILLM,
+        "premai": PremAILLM,
     }
 
     @classmethod
-    def create_llm(cls, type, *args, **kwargs):
+    def create_llm(cls, type, api_key, user_api_key, *args, **kwargs):
         llm_class = cls.llms.get(type.lower())
         if not llm_class:
             raise ValueError(f"No LLM class found for type {type}")
-        return llm_class(*args, **kwargs)
+        return llm_class(api_key, user_api_key, *args, **kwargs)

application/llm/openai.py

@@ -1,36 +1,53 @@
 from application.llm.base import BaseLLM
 from application.core.settings import settings
 
+
 class OpenAILLM(BaseLLM):
 
-    def __init__(self, api_key):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
         global openai
         from openai import OpenAI
-        
+
+        super().__init__(*args, **kwargs)
         self.client = OpenAI(
-                api_key=api_key, 
-            )
+            api_key=api_key,
+        )
         self.api_key = api_key
+        self.user_api_key = user_api_key
 
     def _get_openai(self):
         # Import openai when needed
         import openai
-        
+
         return openai
 
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        response = self.client.chat.completions.create(model=model,
-            messages=messages,
-            stream=stream,
-            **kwargs)
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        engine=settings.AZURE_DEPLOYMENT_NAME,
+        **kwargs
+    ):
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
 
         return response.choices[0].message.content
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        response = self.client.chat.completions.create(model=model,
-            messages=messages,
-            stream=stream,
-            **kwargs)
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        engine=settings.AZURE_DEPLOYMENT_NAME,
+        **kwargs
+    ):
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
 
         for line in response:
             # import sys
@@ -41,14 +58,17 @@ class OpenAILLM(BaseLLM):
 
 class AzureOpenAILLM(OpenAILLM):
 
-    def __init__(self, openai_api_key, openai_api_base, openai_api_version, deployment_name):
+    def __init__(
+        self, openai_api_key, openai_api_base, openai_api_version, deployment_name
+    ):
         super().__init__(openai_api_key)
-        self.api_base = settings.OPENAI_API_BASE,
-        self.api_version = settings.OPENAI_API_VERSION,
-        self.deployment_name = settings.AZURE_DEPLOYMENT_NAME,
+        self.api_base = (settings.OPENAI_API_BASE,)
+        self.api_version = (settings.OPENAI_API_VERSION,)
+        self.deployment_name = (settings.AZURE_DEPLOYMENT_NAME,)
         from openai import AzureOpenAI
+
         self.client = AzureOpenAI(
-            api_key=openai_api_key,  
+            api_key=openai_api_key,
             api_version=settings.OPENAI_API_VERSION,
             api_base=settings.OPENAI_API_BASE,
             deployment_name=settings.AZURE_DEPLOYMENT_NAME,

application/llm/premai.py

@@ -1,32 +1,37 @@
 from application.llm.base import BaseLLM
 from application.core.settings import settings
 
+
 class PremAILLM(BaseLLM):
 
-    def __init__(self, api_key):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
         from premai import Prem
-        
-        self.client = Prem(
-            api_key=api_key
-        )
+
+        super().__init__(*args, **kwargs)
+        self.client = Prem(api_key=api_key)
         self.api_key = api_key
+        self.user_api_key = user_api_key
         self.project_id = settings.PREMAI_PROJECT_ID
 
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        response = self.client.chat.completions.create(model=model,
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        response = self.client.chat.completions.create(
+            model=model,
             project_id=self.project_id,
             messages=messages,
             stream=stream,
-            **kwargs)
+            **kwargs
+        )
 
         return response.choices[0].message["content"]
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        response = self.client.chat.completions.create(model=model,
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+        response = self.client.chat.completions.create(
+            model=model,
             project_id=self.project_id,
             messages=messages,
             stream=stream,
-            **kwargs)
+            **kwargs
+        )
 
         for line in response:
             if line.choices[0].delta["content"] is not None:

application/llm/sagemaker.py

@@ -4,11 +4,10 @@ import json
 import io
 
 
-
 class LineIterator:
     """
-    A helper class for parsing the byte stream input. 
-    
+    A helper class for parsing the byte stream input.
+
     The output of the model will be in the following format:
     ```
     b'{"outputs": [" a"]}\n'
@@ -16,21 +15,21 @@ class LineIterator:
     b'{"outputs": [" problem"]}\n'
     ...
     ```
-    
-    While usually each PayloadPart event from the event stream will contain a byte array 
+
+    While usually each PayloadPart event from the event stream will contain a byte array
     with a full json, this is not guaranteed and some of the json objects may be split across
     PayloadPart events. For example:
     ```
     {'PayloadPart': {'Bytes': b'{"outputs": '}}
     {'PayloadPart': {'Bytes': b'[" problem"]}\n'}}
     ```
-    
+
     This class accounts for this by concatenating bytes written via the 'write' function
     and then exposing a method which will return lines (ending with a '\n' character) within
-    the buffer via the 'scan_lines' function. It maintains the position of the last read 
-    position to ensure that previous bytes are not exposed again. 
+    the buffer via the 'scan_lines' function. It maintains the position of the last read
+    position to ensure that previous bytes are not exposed again.
     """
-    
+
     def __init__(self, stream):
         self.byte_iterator = iter(stream)
         self.buffer = io.BytesIO()
@@ -43,7 +42,7 @@ class LineIterator:
         while True:
             self.buffer.seek(self.read_pos)
             line = self.buffer.readline()
-            if line and line[-1] == ord('\n'):
+            if line and line[-1] == ord("\n"):
                 self.read_pos += len(line)
                 return line[:-1]
             try:
@@ -52,33 +51,35 @@ class LineIterator:
                 if self.read_pos < self.buffer.getbuffer().nbytes:
                     continue
                 raise
-            if 'PayloadPart' not in chunk:
-                print('Unknown event type:' + chunk)
+            if "PayloadPart" not in chunk:
+                print("Unknown event type:" + chunk)
                 continue
             self.buffer.seek(0, io.SEEK_END)
-            self.buffer.write(chunk['PayloadPart']['Bytes'])
+            self.buffer.write(chunk["PayloadPart"]["Bytes"])
+
 
 class SagemakerAPILLM(BaseLLM):
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
         import boto3
+
         runtime = boto3.client(
-            'runtime.sagemaker',
-            aws_access_key_id='xxx',
-            aws_secret_access_key='xxx',
-            region_name='us-west-2'
+            "runtime.sagemaker",
+            aws_access_key_id="xxx",
+            aws_secret_access_key="xxx",
+            region_name="us-west-2",
         )
 
-        
-        self.endpoint =  settings.SAGEMAKER_ENDPOINT
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.endpoint = settings.SAGEMAKER_ENDPOINT
         self.runtime = runtime
 
-
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-    
 
         # Construct payload for endpoint
         payload = {
@@ -89,25 +90,25 @@ class SagemakerAPILLM(BaseLLM):
                 "temperature": 0.1,
                 "max_new_tokens": 30,
                 "repetition_penalty": 1.03,
-                "stop": ["</s>", "###"]
-            }
+                "stop": ["</s>", "###"],
+            },
         }
-        body_bytes = json.dumps(payload).encode('utf-8')
+        body_bytes = json.dumps(payload).encode("utf-8")
 
         # Invoke the endpoint
-        response = self.runtime.invoke_endpoint(EndpointName=self.endpoint,
-                                        ContentType='application/json',
-                                        Body=body_bytes)
-        result = json.loads(response['Body'].read().decode())
+        response = self.runtime.invoke_endpoint(
+            EndpointName=self.endpoint, ContentType="application/json", Body=body_bytes
+        )
+        result = json.loads(response["Body"].read().decode())
         import sys
-        print(result[0]['generated_text'], file=sys.stderr)
-        return result[0]['generated_text'][len(prompt):]
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+        print(result[0]["generated_text"], file=sys.stderr)
+        return result[0]["generated_text"][len(prompt) :]
+
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-    
 
         # Construct payload for endpoint
         payload = {
@@ -118,22 +119,22 @@ class SagemakerAPILLM(BaseLLM):
                 "temperature": 0.1,
                 "max_new_tokens": 512,
                 "repetition_penalty": 1.03,
-                "stop": ["</s>", "###"]
-            }
+                "stop": ["</s>", "###"],
+            },
         }
-        body_bytes = json.dumps(payload).encode('utf-8')
+        body_bytes = json.dumps(payload).encode("utf-8")
 
         # Invoke the endpoint
-        response = self.runtime.invoke_endpoint_with_response_stream(EndpointName=self.endpoint,
-                                        ContentType='application/json',
-                                        Body=body_bytes)
-        #result = json.loads(response['Body'].read().decode())
-        event_stream = response['Body']
-        start_json = b'{'
+        response = self.runtime.invoke_endpoint_with_response_stream(
+            EndpointName=self.endpoint, ContentType="application/json", Body=body_bytes
+        )
+        # result = json.loads(response['Body'].read().decode())
+        event_stream = response["Body"]
+        start_json = b"{"
         for line in LineIterator(event_stream):
-            if line != b'' and start_json in line:
-                #print(line)
-                data = json.loads(line[line.find(start_json):].decode('utf-8'))
-                if data['token']['text'] not in ["</s>", "###"]:
-                    print(data['token']['text'],end='')
-                    yield data['token']['text']
+            if line != b"" and start_json in line:
+                # print(line)
+                data = json.loads(line[line.find(start_json) :].decode("utf-8"))
+                if data["token"]["text"] not in ["</s>", "###"]:
+                    print(data["token"]["text"], end="")
+                    yield data["token"]["text"]

application/parser/remote/reddit_loader.py

@@ -0,0 +1,26 @@
+from application.parser.remote.base import BaseRemote
+from langchain_community.document_loaders import RedditPostsLoader
+
+
+class RedditPostsLoaderRemote(BaseRemote):
+    def load_data(self, inputs):
+        data = eval(inputs)
+        client_id = data.get("client_id")
+        client_secret = data.get("client_secret")
+        user_agent = data.get("user_agent")
+        categories = data.get("categories", ["new", "hot"])
+        mode = data.get("mode", "subreddit")
+        search_queries = data.get("search_queries")
+        number_posts = data.get("number_posts", 10)
+        self.loader = RedditPostsLoader(
+            client_id=client_id,
+            client_secret=client_secret,
+            user_agent=user_agent,
+            categories=categories,
+            mode=mode,
+            search_queries=search_queries,
+            number_posts=number_posts,
+        )
+        documents = self.loader.load()
+        print(f"Loaded {len(documents)} documents from Reddit")
+        return documents

application/parser/remote/remote_creator.py

@@ -1,13 +1,15 @@
 from application.parser.remote.sitemap_loader import SitemapLoader
 from application.parser.remote.crawler_loader import CrawlerLoader
 from application.parser.remote.web_loader import WebLoader
+from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
 
 
 class RemoteCreator:
     loaders = {
-        'url': WebLoader,
-        'sitemap': SitemapLoader,
-        'crawler': CrawlerLoader
+        "url": WebLoader,
+        "sitemap": SitemapLoader,
+        "crawler": CrawlerLoader,
+        "reddit": RedditPostsLoaderRemote,
     }
 
     @classmethod
@@ -15,4 +17,4 @@ class RemoteCreator:
         loader_class = cls.loaders.get(type.lower())
         if not loader_class:
             raise ValueError(f"No LLM class found for type {type}")
-        return loader_class(*args, **kwargs)
+        return loader_class(*args, **kwargs)

application/parser/token_func.py

@@ -22,7 +22,10 @@ def group_documents(documents: List[Document], min_tokens: int, max_tokens: int)
         doc_len = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
 
         # Check if current group is empty or if the document can be added based on token count and matching metadata
-        if current_group is None or (len(tiktoken.get_encoding("cl100k_base").encode(current_group.text)) + doc_len < max_tokens and doc_len < min_tokens and current_group.extra_info == doc.extra_info):
+        if (current_group is None or 
+            (len(tiktoken.get_encoding("cl100k_base").encode(current_group.text)) + doc_len < max_tokens and 
+             doc_len < min_tokens and 
+             current_group.extra_info == doc.extra_info)):
             if current_group is None:
                 current_group = doc  # Use the document directly to retain its metadata
             else:

application/requirements.txt

@@ -3,6 +3,7 @@ boto3==1.34.6
 celery==5.3.6
 dataclasses_json==0.6.3
 docx2txt==0.8
+duckduckgo-search==5.3.0
 EbookLib==0.18
 elasticsearch==8.12.0
 escodegen==1.0.11
@@ -18,10 +19,10 @@ nltk==3.8.1
 openapi3_parser==1.1.16
 pandas==2.2.0
 pydantic_settings==2.1.0
-pymongo==4.6.1
+pymongo==4.6.3
 PyPDF2==3.0.1
 python-dotenv==1.0.1
-qdrant-client==1.7.3
+qdrant-client==1.8.2
 redis==5.0.1
 Requests==2.31.0
 retry==0.9.2

application/retriever/base.py

@@ -0,0 +1,14 @@
+from abc import ABC, abstractmethod
+
+
+class BaseRetriever(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def gen(self, *args, **kwargs):
+        pass
+
+    @abstractmethod
+    def search(self, *args, **kwargs):
+        pass

application/retriever/brave_search.py

@@ -0,0 +1,95 @@
+import json
+from application.retriever.base import BaseRetriever
+from application.core.settings import settings
+from application.llm.llm_creator import LLMCreator
+from application.utils import count_tokens
+from langchain_community.tools import BraveSearch
+
+
+class BraveRetSearch(BaseRetriever):
+
+    def __init__(
+        self,
+        question,
+        source,
+        chat_history,
+        prompt,
+        chunks=2,
+        gpt_model="docsgpt",
+        user_api_key=None,
+    ):
+        self.question = question
+        self.source = source
+        self.chat_history = chat_history
+        self.prompt = prompt
+        self.chunks = chunks
+        self.gpt_model = gpt_model
+        self.user_api_key = user_api_key
+
+    def _get_data(self):
+        if self.chunks == 0:
+            docs = []
+        else:
+            search = BraveSearch.from_api_key(
+                api_key=settings.BRAVE_SEARCH_API_KEY,
+                search_kwargs={"count": int(self.chunks)},
+            )
+            results = search.run(self.question)
+            results = json.loads(results)
+
+            docs = []
+            for i in results:
+                try:
+                    title = i["title"]
+                    link = i["link"]
+                    snippet = i["snippet"]
+                    docs.append({"text": snippet, "title": title, "link": link})
+                except IndexError:
+                    pass
+        if settings.LLM_NAME == "llama.cpp":
+            docs = [docs[0]]
+
+        return docs
+
+    def gen(self):
+        docs = self._get_data()
+
+        # join all page_content together with a newline
+        docs_together = "\n".join([doc["text"] for doc in docs])
+        p_chat_combine = self.prompt.replace("{summaries}", docs_together)
+        messages_combine = [{"role": "system", "content": p_chat_combine}]
+        for doc in docs:
+            yield {"source": doc}
+
+        if len(self.chat_history) > 1:
+            tokens_current_history = 0
+            # count tokens in history
+            self.chat_history.reverse()
+            for i in self.chat_history:
+                if "prompt" in i and "response" in i:
+                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
+                        i["response"]
+                    )
+                    if (
+                        tokens_current_history + tokens_batch
+                        < settings.TOKENS_MAX_HISTORY
+                    ):
+                        tokens_current_history += tokens_batch
+                        messages_combine.append(
+                            {"role": "user", "content": i["prompt"]}
+                        )
+                        messages_combine.append(
+                            {"role": "system", "content": i["response"]}
+                        )
+        messages_combine.append({"role": "user", "content": self.question})
+
+        llm = LLMCreator.create_llm(
+            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        )
+
+        completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        for line in completion:
+            yield {"answer": str(line)}
+
+    def search(self):
+        return self._get_data()

application/retriever/classic_rag.py

@@ -0,0 +1,110 @@
+import os
+from application.retriever.base import BaseRetriever
+from application.core.settings import settings
+from application.vectorstore.vector_creator import VectorCreator
+from application.llm.llm_creator import LLMCreator
+
+from application.utils import count_tokens
+
+
+class ClassicRAG(BaseRetriever):
+
+    def __init__(
+        self,
+        question,
+        source,
+        chat_history,
+        prompt,
+        chunks=2,
+        gpt_model="docsgpt",
+        user_api_key=None,
+    ):
+        self.question = question
+        self.vectorstore = self._get_vectorstore(source=source)
+        self.chat_history = chat_history
+        self.prompt = prompt
+        self.chunks = chunks
+        self.gpt_model = gpt_model
+        self.user_api_key = user_api_key
+
+    def _get_vectorstore(self, source):
+        if "active_docs" in source:
+            if source["active_docs"].split("/")[0] == "default":
+                vectorstore = ""
+            elif source["active_docs"].split("/")[0] == "local":
+                vectorstore = "indexes/" + source["active_docs"]
+            else:
+                vectorstore = "vectors/" + source["active_docs"]
+            if source["active_docs"] == "default":
+                vectorstore = ""
+        else:
+            vectorstore = ""
+        vectorstore = os.path.join("application", vectorstore)
+        return vectorstore
+
+    def _get_data(self):
+        if self.chunks == 0:
+            docs = []
+        else:
+            docsearch = VectorCreator.create_vectorstore(
+                settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
+            )
+            docs_temp = docsearch.search(self.question, k=self.chunks)
+            docs = [
+                {
+                    "title": (
+                        i.metadata["title"].split("/")[-1]
+                        if i.metadata
+                        else i.page_content
+                    ),
+                    "text": i.page_content,
+                }
+                for i in docs_temp
+            ]
+        if settings.LLM_NAME == "llama.cpp":
+            docs = [docs[0]]
+
+        return docs
+
+    def gen(self):
+        docs = self._get_data()
+
+        # join all page_content together with a newline
+        docs_together = "\n".join([doc["text"] for doc in docs])
+        p_chat_combine = self.prompt.replace("{summaries}", docs_together)
+        messages_combine = [{"role": "system", "content": p_chat_combine}]
+        for doc in docs:
+            yield {"source": doc}
+
+        if len(self.chat_history) > 1:
+            tokens_current_history = 0
+            # count tokens in history
+            self.chat_history.reverse()
+            for i in self.chat_history:
+                if "prompt" in i and "response" in i:
+                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
+                        i["response"]
+                    )
+                    if (
+                        tokens_current_history + tokens_batch
+                        < settings.TOKENS_MAX_HISTORY
+                    ):
+                        tokens_current_history += tokens_batch
+                        messages_combine.append(
+                            {"role": "user", "content": i["prompt"]}
+                        )
+                        messages_combine.append(
+                            {"role": "system", "content": i["response"]}
+                        )
+        messages_combine.append({"role": "user", "content": self.question})
+
+        llm = LLMCreator.create_llm(
+            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        )
+
+        completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        for line in completion:
+            yield {"answer": str(line)}
+
+    def search(self):
+        return self._get_data()

application/retriever/duckduck_search.py

@@ -0,0 +1,112 @@
+from application.retriever.base import BaseRetriever
+from application.core.settings import settings
+from application.llm.llm_creator import LLMCreator
+from application.utils import count_tokens
+from langchain_community.tools import DuckDuckGoSearchResults
+from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
+
+
+class DuckDuckSearch(BaseRetriever):
+
+    def __init__(
+        self,
+        question,
+        source,
+        chat_history,
+        prompt,
+        chunks=2,
+        gpt_model="docsgpt",
+        user_api_key=None,
+    ):
+        self.question = question
+        self.source = source
+        self.chat_history = chat_history
+        self.prompt = prompt
+        self.chunks = chunks
+        self.gpt_model = gpt_model
+        self.user_api_key = user_api_key
+
+    def _parse_lang_string(self, input_string):
+        result = []
+        current_item = ""
+        inside_brackets = False
+        for char in input_string:
+            if char == "[":
+                inside_brackets = True
+            elif char == "]":
+                inside_brackets = False
+                result.append(current_item)
+                current_item = ""
+            elif inside_brackets:
+                current_item += char
+
+        if inside_brackets:
+            result.append(current_item)
+
+        return result
+
+    def _get_data(self):
+        if self.chunks == 0:
+            docs = []
+        else:
+            wrapper = DuckDuckGoSearchAPIWrapper(max_results=self.chunks)
+            search = DuckDuckGoSearchResults(api_wrapper=wrapper)
+            results = search.run(self.question)
+            results = self._parse_lang_string(results)
+
+            docs = []
+            for i in results:
+                try:
+                    text = i.split("title:")[0]
+                    title = i.split("title:")[1].split("link:")[0]
+                    link = i.split("link:")[1]
+                    docs.append({"text": text, "title": title, "link": link})
+                except IndexError:
+                    pass
+        if settings.LLM_NAME == "llama.cpp":
+            docs = [docs[0]]
+
+        return docs
+
+    def gen(self):
+        docs = self._get_data()
+
+        # join all page_content together with a newline
+        docs_together = "\n".join([doc["text"] for doc in docs])
+        p_chat_combine = self.prompt.replace("{summaries}", docs_together)
+        messages_combine = [{"role": "system", "content": p_chat_combine}]
+        for doc in docs:
+            yield {"source": doc}
+
+        if len(self.chat_history) > 1:
+            tokens_current_history = 0
+            # count tokens in history
+            self.chat_history.reverse()
+            for i in self.chat_history:
+                if "prompt" in i and "response" in i:
+                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
+                        i["response"]
+                    )
+                    if (
+                        tokens_current_history + tokens_batch
+                        < settings.TOKENS_MAX_HISTORY
+                    ):
+                        tokens_current_history += tokens_batch
+                        messages_combine.append(
+                            {"role": "user", "content": i["prompt"]}
+                        )
+                        messages_combine.append(
+                            {"role": "system", "content": i["response"]}
+                        )
+        messages_combine.append({"role": "user", "content": self.question})
+
+        llm = LLMCreator.create_llm(
+            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        )
+
+        completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        for line in completion:
+            yield {"answer": str(line)}
+
+    def search(self):
+        return self._get_data()

application/retriever/retriever_creator.py

@@ -0,0 +1,19 @@
+from application.retriever.classic_rag import ClassicRAG
+from application.retriever.duckduck_search import DuckDuckSearch
+from application.retriever.brave_search import BraveRetSearch
+
+
+
+class RetrieverCreator:
+    retievers = {
+        'classic': ClassicRAG,
+        'duckduck_search': DuckDuckSearch,
+        'brave_search': BraveRetSearch
+    }
+
+    @classmethod
+    def create_retriever(cls, type, *args, **kwargs):
+        retiever_class = cls.retievers.get(type.lower())
+        if not retiever_class:
+            raise ValueError(f"No retievers class found for type {type}")
+        return retiever_class(*args, **kwargs)

application/usage.py

@@ -0,0 +1,49 @@
+import sys
+from pymongo import MongoClient
+from datetime import datetime
+from application.core.settings import settings
+from application.utils import count_tokens
+
+mongo = MongoClient(settings.MONGO_URI)
+db = mongo["docsgpt"]
+usage_collection = db["token_usage"]
+
+
+def update_token_usage(user_api_key, token_usage):
+    if "pytest" in sys.modules:
+        return
+    usage_data = {
+        "api_key": user_api_key,
+        "prompt_tokens": token_usage["prompt_tokens"],
+        "generated_tokens": token_usage["generated_tokens"],
+        "timestamp": datetime.now(),
+    }
+    usage_collection.insert_one(usage_data)
+
+
+def gen_token_usage(func):
+    def wrapper(self, model, messages, stream, **kwargs):
+        for message in messages:
+            self.token_usage["prompt_tokens"] += count_tokens(message["content"])
+        result = func(self, model, messages, stream, **kwargs)
+        self.token_usage["generated_tokens"] += count_tokens(result)
+        update_token_usage(self.user_api_key, self.token_usage)
+        return result
+
+    return wrapper
+
+
+def stream_token_usage(func):
+    def wrapper(self, model, messages, stream, **kwargs):
+        for message in messages:
+            self.token_usage["prompt_tokens"] += count_tokens(message["content"])
+        batch = []
+        result = func(self, model, messages, stream, **kwargs)
+        for r in result:
+            batch.append(r)
+            yield r
+        for line in batch:
+            self.token_usage["generated_tokens"] += count_tokens(line)
+        update_token_usage(self.user_api_key, self.token_usage)
+
+    return wrapper

application/utils.py

@@ -0,0 +1,6 @@
+from transformers import GPT2TokenizerFast
+
+
+def count_tokens(string):
+    tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
+    return len(tokenizer(string)['input_ids'])

application/worker.py

@@ -15,23 +15,53 @@ from application.parser.schema.base import Document
 from application.parser.token_func import group_split
 
 try:
-    nltk.download('punkt', quiet=True)
-    nltk.download('averaged_perceptron_tagger', quiet=True)
+    nltk.download("punkt", quiet=True)
+    nltk.download("averaged_perceptron_tagger", quiet=True)
 except FileExistsError:
     pass
 
 
 # Define a function to extract metadata from a given filename.
 def metadata_from_filename(title):
-    store = '/'.join(title.split('/')[1:3])
-    return {'title': title, 'store': store}
+    store = "/".join(title.split("/")[1:3])
+    return {"title": title, "store": store}
 
 
 # Define a function to generate a random string of a given length.
 def generate_random_string(length):
-    return ''.join([string.ascii_letters[i % 52] for i in range(length)])
+    return "".join([string.ascii_letters[i % 52] for i in range(length)])
+
+
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
+    """
+    Recursively extract zip files with a limit on recursion depth.
+
+    Args:
+        zip_path (str): Path to the zip file to be extracted.
+        extract_to (str): Destination path for extracted files.
+        current_depth (int): Current depth of recursion.
+        max_depth (int): Maximum allowed depth of recursion to prevent infinite loops.
+    """
+    if current_depth > max_depth:
+        print(f"Reached maximum recursion depth of {max_depth}")
+        return
+
+    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+        zip_ref.extractall(extract_to)
+    os.remove(zip_path)  # Remove the zip file after extracting
+
+    # Check for nested zip files and extract them
+    for root, dirs, files in os.walk(extract_to):
+        for file in files:
+            if file.endswith(".zip"):
+                # If a nested zip file is found, extract it recursively
+                file_path = os.path.join(root, file)
+                extract_zip_recursive(file_path, root, current_depth + 1, max_depth)
 
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 # Define the main function for ingesting and processing documents.
 def ingest_worker(self, directory, formats, name_job, filename, user):
@@ -62,38 +92,52 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
     token_check = True
     min_tokens = 150
     max_tokens = 1250
-    full_path = directory + '/' + user + '/' + name_job
+    recursion_depth = 2
+    full_path = os.path.join(directory, user, name_job)
     import sys
+
+
     print(full_path, file=sys.stderr)
     # check if API_URL env variable is set
-    file_data = {'name': name_job, 'file': filename, 'user': user}
-    response = requests.get(urljoin(settings.API_URL, "/api/download"), params=file_data)
+    file_data = {"name": name_job, "file": filename, "user": user}
+    response = requests.get(
+        urljoin(settings.API_URL, "/api/download"), params=file_data
+    )
     # check if file is in the response
     print(response, file=sys.stderr)
     file = response.content
 
     if not os.path.exists(full_path):
         os.makedirs(full_path)
-    with open(full_path + '/' + filename, 'wb') as f:
+    with open(os.path.join(full_path, filename), "wb") as f:
         f.write(file)
 
     # check if file is .zip and extract it
-    if filename.endswith('.zip'):
-        with zipfile.ZipFile(full_path + '/' + filename, 'r') as zip_ref:
-            zip_ref.extractall(full_path)
-        os.remove(full_path + '/' + filename)
+    if filename.endswith(".zip"):
+        extract_zip_recursive(os.path.join(full_path, filename), full_path, 0, recursion_depth)
 
-    self.update_state(state='PROGRESS', meta={'current': 1})
+    self.update_state(state="PROGRESS", meta={"current": 1})
 
-    raw_docs = SimpleDirectoryReader(input_dir=full_path, input_files=input_files, recursive=recursive,
-                                     required_exts=formats, num_files_limit=limit,
-                                     exclude_hidden=exclude, file_metadata=metadata_from_filename).load_data()
-    raw_docs = group_split(documents=raw_docs, min_tokens=min_tokens, max_tokens=max_tokens, token_check=token_check)
+    raw_docs = SimpleDirectoryReader(
+        input_dir=full_path,
+        input_files=input_files,
+        recursive=recursive,
+        required_exts=formats,
+        num_files_limit=limit,
+        exclude_hidden=exclude,
+        file_metadata=metadata_from_filename,
+    ).load_data()
+    raw_docs = group_split(
+        documents=raw_docs,
+        min_tokens=min_tokens,
+        max_tokens=max_tokens,
+        token_check=token_check,
+    )
 
     docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
 
     call_openai_api(docs, full_path, self)
-    self.update_state(state='PROGRESS', meta={'current': 100})
+    self.update_state(state="PROGRESS", meta={"current": 100})
 
     if sample:
         for i in range(min(5, len(raw_docs))):
@@ -101,70 +145,80 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
 
     # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
     # and send them to the server (provide user and name in form)
-    file_data = {'name': name_job, 'user': user}
+    file_data = {"name": name_job, "user": user}
     if settings.VECTOR_STORE == "faiss":
-        files = {'file_faiss': open(full_path + '/index.faiss', 'rb'),
-                'file_pkl': open(full_path + '/index.pkl', 'rb')}
-        response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data)
-        response = requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path))
+        files = {
+            "file_faiss": open(full_path + "/index.faiss", "rb"),
+            "file_pkl": open(full_path + "/index.pkl", "rb"),
+        }
+        response = requests.post(
+            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
+        )
+        response = requests.get(
+            urljoin(settings.API_URL, "/api/delete_old?path=" + full_path)
+        )
     else:
-        response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
+        response = requests.post(
+            urljoin(settings.API_URL, "/api/upload_index"), data=file_data
+        )
 
-    
     # delete local
     shutil.rmtree(full_path)
 
     return {
-        'directory': directory,
-        'formats': formats,
-        'name_job': name_job,
-        'filename': filename,
-        'user': user,
-        'limited': False
+        "directory": directory,
+        "formats": formats,
+        "name_job": name_job,
+        "filename": filename,
+        "user": user,
+        "limited": False,
     }
 
-def remote_worker(self, source_data, name_job, user, directory = 'temp', loader = 'url'):
+
+def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
     # sample = False
     token_check = True
     min_tokens = 150
     max_tokens = 1250
-    full_path = directory + '/' + user + '/' + name_job
+    full_path = directory + "/" + user + "/" + name_job
 
     if not os.path.exists(full_path):
         os.makedirs(full_path)
 
-    self.update_state(state='PROGRESS', meta={'current': 1})
- 
+    self.update_state(state="PROGRESS", meta={"current": 1})
+
     # source_data {"data": [url]} for url type task just urls
- 
+
     # Use RemoteCreator to load data from URL
     remote_loader = RemoteCreator.create_loader(loader)
     raw_docs = remote_loader.load_data(source_data)
 
-    docs = group_split(documents=raw_docs, min_tokens=min_tokens, max_tokens=max_tokens, token_check=token_check)
+    docs = group_split(
+        documents=raw_docs,
+        min_tokens=min_tokens,
+        max_tokens=max_tokens,
+        token_check=token_check,
+    )
 
-    #docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+    # docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
 
     call_openai_api(docs, full_path, self)
-    self.update_state(state='PROGRESS', meta={'current': 100})
-    
-
+    self.update_state(state="PROGRESS", meta={"current": 100})
 
     # Proceed with uploading and cleaning as in the original function
-    file_data = {'name': name_job, 'user': user}
+    file_data = {"name": name_job, "user": user}
     if settings.VECTOR_STORE == "faiss":
-        files = {'file_faiss': open(full_path + '/index.faiss', 'rb'),
-                 'file_pkl': open(full_path + '/index.pkl', 'rb')}
-        requests.post(urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data)
+        files = {
+            "file_faiss": open(full_path + "/index.faiss", "rb"),
+            "file_pkl": open(full_path + "/index.pkl", "rb"),
+        }
+        requests.post(
+            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
+        )
         requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path))
     else:
         requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
 
     shutil.rmtree(full_path)
 
-    return {
-        'urls': source_data,
-        'name_job': name_job,
-        'user': user,
-        'limited': False
-    }
+    return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}

application/wsgi.py

@@ -1,4 +1,5 @@
 from application.app import app
+from application.core.settings import settings
 
 if __name__ == "__main__":
-    app.run(debug=True, port=7091)
+    app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)

docs/package-lock.json

@@ -7,7 +7,7 @@
       "license": "MIT",
       "dependencies": {
         "@vercel/analytics": "^1.1.1",
-        "docsgpt": "^0.3.0",
+        "docsgpt": "^0.3.7",
         "next": "^14.0.4",
         "nextra": "^2.13.2",
         "nextra-theme-docs": "^2.13.2",
@@ -422,6 +422,11 @@
         "node": ">=6.9.0"
       }
     },
+    "node_modules/@bpmn-io/snarkdown": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@bpmn-io/snarkdown/-/snarkdown-2.2.0.tgz",
+      "integrity": "sha512-bVD7FIoaBDZeCJkMRgnBPDeptPlto87wt2qaCjf5t8iLaevDmTPaREd6FpBEGsHlUdHFFZWRk4qAoEC5So2M0Q=="
+    },
     "node_modules/@braintree/sanitize-url": {
       "version": "6.0.4",
       "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-6.0.4.tgz",
@@ -4958,11 +4963,12 @@
       }
     },
     "node_modules/docsgpt": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/docsgpt/-/docsgpt-0.3.0.tgz",
-      "integrity": "sha512-0yT2m+HAlJ+289p278c3Zi07bu2wr6zULOT/bYXtJ/nb59V2Vpfdj2xFB49+lYLSeVe8H+Ij5fFSNZ6RkVRfMQ==",
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/docsgpt/-/docsgpt-0.3.7.tgz",
+      "integrity": "sha512-VHrXXOEFtjNTcpA8Blf3IzpLlJxOMhm/S5CM4FDjQEkdK9WWhI8yXd/0Rs/FS8oz7YbFrNxO758mlP7OtQtBBw==",
       "dependencies": {
         "@babel/plugin-transform-flow-strip-types": "^7.23.3",
+        "@bpmn-io/snarkdown": "^2.2.0",
         "@parcel/resolver-glob": "^2.12.0",
         "@parcel/transformer-svg-react": "^2.12.0",
         "@parcel/transformer-typescript-tsc": "^2.12.0",
@@ -4972,6 +4978,7 @@
         "@types/react-dom": "^18.2.19",
         "class-variance-authority": "^0.7.0",
         "clsx": "^2.1.0",
+        "dompurify": "^3.0.9",
         "flow-bin": "^0.229.2",
         "i": "^0.3.7",
         "install": "^0.13.0",
@@ -5029,9 +5036,9 @@
       }
     },
     "node_modules/dompurify": {
-      "version": "3.0.7",
-      "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.7.tgz",
-      "integrity": "sha512-BViYTZoqP3ak/ULKOc101y+CtHDUvBsVgSxIF1ku0HmK6BRf+C03MC+tArMvOPtVtZp83DDh5puywKDu4sbVjQ=="
+      "version": "3.0.11",
+      "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.11.tgz",
+      "integrity": "sha512-Fan4uMuyB26gFV3ovPoEoQbxRRPfTu3CvImyZnhGq5fsIEO+gEFLp45ISFt+kQBWsK5ulDdT0oV28jS1UrwQLg=="
     },
     "node_modules/domutils": {
       "version": "2.8.0",
@@ -6206,9 +6213,9 @@
       "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w=="
     },
     "node_modules/katex": {
-      "version": "0.16.9",
-      "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.9.tgz",
-      "integrity": "sha512-fsSYjWS0EEOwvy81j3vRA8TEAhQhKiqO+FQaKWp0m39qwOzHVBgAUBIXWj1pB+O2W3fIpNa6Y9KSKCVbfPhyAQ==",
+      "version": "0.16.10",
+      "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.10.tgz",
+      "integrity": "sha512-ZiqaC04tp2O5utMsl2TEZTXxa6WSC4yo0fv5ML++D3QZv/vx2Mct0mTlRx3O+uUkjfuAgOkzsCmq5MiUEsDDdA==",
       "funding": [
         "https://opencollective.com/katex",
         "https://github.com/sponsors/katex"
@@ -8136,9 +8143,9 @@
       "integrity": "sha512-gkXMxRzUH+PB0ax9dUN0yYF0S25BqeAYqhgMaLUFmpXLEk7Fcu8f4emJuOAY0V8kjDICxROIKsTAKsV/v355xw=="
     },
     "node_modules/npm": {
-      "version": "10.5.0",
-      "resolved": "https://registry.npmjs.org/npm/-/npm-10.5.0.tgz",
-      "integrity": "sha512-Ejxwvfh9YnWVU2yA5FzoYLTW52vxHCz+MHrOFg9Cc8IFgF/6f5AGPAvb5WTay5DIUP1NIfN3VBZ0cLlGO0Ys+A==",
+      "version": "10.5.1",
+      "resolved": "https://registry.npmjs.org/npm/-/npm-10.5.1.tgz",
+      "integrity": "sha512-RozZuGuWbbhDM2sRhOSLIRb3DLyof6TREi0TW5b3xUEBropDhDqEHv0iAjA1zsIwXKgfIkR8GvQMd4oeKKg9eQ==",
       "bundleDependencies": [
         "@isaacs/string-locale-compare",
         "@npmcli/arborist",
@@ -8147,6 +8154,7 @@
         "@npmcli/map-workspaces",
         "@npmcli/package-json",
         "@npmcli/promise-spawn",
+        "@npmcli/redact",
         "@npmcli/run-script",
         "@sigstore/tuf",
         "abbrev",
@@ -8219,23 +8227,24 @@
         "@npmcli/map-workspaces": "^3.0.4",
         "@npmcli/package-json": "^5.0.0",
         "@npmcli/promise-spawn": "^7.0.1",
+        "@npmcli/redact": "^1.1.0",
         "@npmcli/run-script": "^7.0.4",
-        "@sigstore/tuf": "^2.3.1",
+        "@sigstore/tuf": "^2.3.2",
         "abbrev": "^2.0.0",
         "archy": "~1.0.0",
         "cacache": "^18.0.2",
         "chalk": "^5.3.0",
         "ci-info": "^4.0.0",
         "cli-columns": "^4.0.0",
-        "cli-table3": "^0.6.3",
+        "cli-table3": "^0.6.4",
         "columnify": "^1.6.0",
         "fastest-levenshtein": "^1.0.16",
         "fs-minipass": "^3.0.3",
-        "glob": "^10.3.10",
+        "glob": "^10.3.12",
         "graceful-fs": "^4.2.11",
         "hosted-git-info": "^7.0.1",
-        "ini": "^4.1.1",
-        "init-package-json": "^6.0.0",
+        "ini": "^4.1.2",
+        "init-package-json": "^6.0.2",
         "is-cidr": "^5.0.3",
         "json-parse-even-better-errors": "^3.0.1",
         "libnpmaccess": "^8.0.1",
@@ -8250,11 +8259,11 @@
         "libnpmteam": "^6.0.0",
         "libnpmversion": "^5.0.1",
         "make-fetch-happen": "^13.0.0",
-        "minimatch": "^9.0.3",
+        "minimatch": "^9.0.4",
         "minipass": "^7.0.4",
         "minipass-pipeline": "^1.2.4",
         "ms": "^2.1.2",
-        "node-gyp": "^10.0.1",
+        "node-gyp": "^10.1.0",
         "nopt": "^7.2.0",
         "normalize-package-data": "^6.0.0",
         "npm-audit-report": "^5.0.0",
@@ -8262,7 +8271,7 @@
         "npm-package-arg": "^11.0.1",
         "npm-pick-manifest": "^9.0.0",
         "npm-profile": "^9.0.0",
-        "npm-registry-fetch": "^16.1.0",
+        "npm-registry-fetch": "^16.2.0",
         "npm-user-validate": "^2.0.0",
         "npmlog": "^7.0.1",
         "p-map": "^4.0.0",
@@ -8270,12 +8279,12 @@
         "parse-conflict-json": "^3.0.1",
         "proc-log": "^3.0.0",
         "qrcode-terminal": "^0.12.0",
-        "read": "^2.1.0",
+        "read": "^3.0.1",
         "semver": "^7.6.0",
         "spdx-expression-parse": "^3.0.1",
         "ssri": "^10.0.5",
         "supports-color": "^9.4.0",
-        "tar": "^6.2.0",
+        "tar": "^6.2.1",
         "text-table": "~0.2.0",
         "tiny-relative-date": "^1.3.0",
         "treeverse": "^3.0.0",
@@ -8332,8 +8341,6 @@
     },
     "node_modules/npm/node_modules/@isaacs/cliui": {
       "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
-      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
@@ -8350,8 +8357,6 @@
     },
     "node_modules/npm/node_modules/@isaacs/cliui/node_modules/ansi-regex": {
       "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
-      "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
       "inBundle": true,
       "license": "MIT",
       "engines": {
@@ -8363,8 +8368,6 @@
     },
     "node_modules/npm/node_modules/@isaacs/cliui/node_modules/emoji-regex": {
       "version": "9.2.2",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
-      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==",
       "inBundle": true,
       "license": "MIT"
     },
@@ -8386,8 +8389,6 @@
     },
     "node_modules/npm/node_modules/@isaacs/cliui/node_modules/strip-ansi": {
       "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
-      "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -8421,7 +8422,7 @@
       }
     },
     "node_modules/npm/node_modules/@npmcli/arborist": {
-      "version": "7.4.0",
+      "version": "7.4.1",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
@@ -8441,12 +8442,12 @@
         "hosted-git-info": "^7.0.1",
         "json-parse-even-better-errors": "^3.0.0",
         "json-stringify-nice": "^1.1.4",
-        "minimatch": "^9.0.0",
+        "minimatch": "^9.0.4",
         "nopt": "^7.0.0",
         "npm-install-checks": "^6.2.0",
         "npm-package-arg": "^11.0.1",
         "npm-pick-manifest": "^9.0.0",
-        "npm-registry-fetch": "^16.0.0",
+        "npm-registry-fetch": "^16.2.0",
         "npmlog": "^7.0.1",
         "pacote": "^17.0.4",
         "parse-conflict-json": "^3.0.0",
@@ -8467,13 +8468,13 @@
       }
     },
     "node_modules/npm/node_modules/@npmcli/config": {
-      "version": "8.2.0",
+      "version": "8.2.1",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
         "@npmcli/map-workspaces": "^3.0.2",
         "ci-info": "^4.0.0",
-        "ini": "^4.1.0",
+        "ini": "^4.1.2",
         "nopt": "^7.0.0",
         "proc-log": "^3.0.0",
         "read-package-json-fast": "^3.0.2",
@@ -8636,6 +8637,14 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
+    "node_modules/npm/node_modules/@npmcli/redact": {
+      "version": "1.1.0",
+      "inBundle": true,
+      "license": "ISC",
+      "engines": {
+        "node": "^16.14.0 || >=18.0.0"
+      }
+    },
     "node_modules/npm/node_modules/@npmcli/run-script": {
       "version": "7.0.4",
       "inBundle": true,
@@ -8653,8 +8662,6 @@
     },
     "node_modules/npm/node_modules/@pkgjs/parseargs": {
       "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
-      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
       "inBundle": true,
       "license": "MIT",
       "optional": true,
@@ -8704,7 +8711,7 @@
       }
     },
     "node_modules/npm/node_modules/@sigstore/tuf": {
-      "version": "2.3.1",
+      "version": "2.3.2",
       "inBundle": true,
       "license": "Apache-2.0",
       "dependencies": {
@@ -8757,7 +8764,7 @@
       }
     },
     "node_modules/npm/node_modules/agent-base": {
-      "version": "7.1.0",
+      "version": "7.1.1",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -8781,8 +8788,6 @@
     },
     "node_modules/npm/node_modules/ansi-regex": {
       "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
       "inBundle": true,
       "license": "MIT",
       "engines": {
@@ -8791,8 +8796,6 @@
     },
     "node_modules/npm/node_modules/ansi-styles": {
       "version": "6.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
-      "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
       "inBundle": true,
       "license": "MIT",
       "engines": {
@@ -8822,8 +8825,6 @@
     },
     "node_modules/npm/node_modules/balanced-match": {
       "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
-      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
       "inBundle": true,
       "license": "MIT"
     },
@@ -8842,17 +8843,18 @@
       }
     },
     "node_modules/npm/node_modules/binary-extensions": {
-      "version": "2.2.0",
+      "version": "2.3.0",
       "inBundle": true,
       "license": "MIT",
       "engines": {
         "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
     "node_modules/npm/node_modules/brace-expansion": {
       "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -8954,7 +8956,7 @@
       }
     },
     "node_modules/npm/node_modules/cli-table3": {
-      "version": "0.6.3",
+      "version": "0.6.4",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -8985,8 +8987,6 @@
     },
     "node_modules/npm/node_modules/color-convert": {
       "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -8998,8 +8998,6 @@
     },
     "node_modules/npm/node_modules/color-name": {
       "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "inBundle": true,
       "license": "MIT"
     },
@@ -9035,8 +9033,6 @@
     },
     "node_modules/npm/node_modules/cross-spawn": {
       "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
-      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -9064,8 +9060,6 @@
     },
     "node_modules/npm/node_modules/cssesc": {
       "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
-      "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==",
       "inBundle": true,
       "license": "MIT",
       "bin": {
@@ -9117,15 +9111,11 @@
     },
     "node_modules/npm/node_modules/eastasianwidth": {
       "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
-      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
       "inBundle": true,
       "license": "MIT"
     },
     "node_modules/npm/node_modules/emoji-regex": {
       "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
       "inBundle": true,
       "license": "MIT"
     },
@@ -9166,8 +9156,6 @@
     },
     "node_modules/npm/node_modules/foreground-child": {
       "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
-      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
@@ -9194,8 +9182,6 @@
     },
     "node_modules/npm/node_modules/function-bind": {
       "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
-      "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
       "inBundle": true,
       "license": "MIT",
       "funding": {
@@ -9221,17 +9207,15 @@
       }
     },
     "node_modules/npm/node_modules/glob": {
-      "version": "10.3.10",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz",
-      "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==",
+      "version": "10.3.12",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
         "foreground-child": "^3.1.0",
-        "jackspeak": "^2.3.5",
+        "jackspeak": "^2.3.6",
         "minimatch": "^9.0.1",
-        "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0",
-        "path-scurry": "^1.10.1"
+        "minipass": "^7.0.4",
+        "path-scurry": "^1.10.2"
       },
       "bin": {
         "glob": "dist/esm/bin.mjs"
@@ -9245,8 +9229,6 @@
     },
     "node_modules/npm/node_modules/graceful-fs": {
       "version": "4.2.11",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
-      "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
       "inBundle": true,
       "license": "ISC"
     },
@@ -9346,7 +9328,7 @@
       }
     },
     "node_modules/npm/node_modules/ini": {
-      "version": "4.1.1",
+      "version": "4.1.2",
       "inBundle": true,
       "license": "ISC",
       "engines": {
@@ -9354,14 +9336,14 @@
       }
     },
     "node_modules/npm/node_modules/init-package-json": {
-      "version": "6.0.0",
+      "version": "6.0.2",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
+        "@npmcli/package-json": "^5.0.0",
         "npm-package-arg": "^11.0.0",
         "promzard": "^1.0.0",
-        "read": "^2.0.0",
-        "read-package-json": "^7.0.0",
+        "read": "^3.0.1",
         "semver": "^7.3.5",
         "validate-npm-package-license": "^3.0.4",
         "validate-npm-package-name": "^5.0.0"
@@ -9422,8 +9404,6 @@
     },
     "node_modules/npm/node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
-      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
       "inBundle": true,
       "license": "MIT",
       "engines": {
@@ -9437,15 +9417,11 @@
     },
     "node_modules/npm/node_modules/isexe": {
       "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
       "inBundle": true,
       "license": "ISC"
     },
     "node_modules/npm/node_modules/jackspeak": {
       "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",
-      "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==",
       "inBundle": true,
       "license": "BlueOak-1.0.0",
       "dependencies": {
@@ -9501,38 +9477,38 @@
       "license": "MIT"
     },
     "node_modules/npm/node_modules/libnpmaccess": {
-      "version": "8.0.2",
+      "version": "8.0.3",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
         "npm-package-arg": "^11.0.1",
-        "npm-registry-fetch": "^16.0.0"
+        "npm-registry-fetch": "^16.2.0"
       },
       "engines": {
         "node": "^16.14.0 || >=18.0.0"
       }
     },
     "node_modules/npm/node_modules/libnpmdiff": {
-      "version": "6.0.7",
+      "version": "6.0.8",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
         "@npmcli/arborist": "^7.2.1",
         "@npmcli/disparity-colors": "^3.0.0",
         "@npmcli/installed-package-contents": "^2.0.2",
-        "binary-extensions": "^2.2.0",
+        "binary-extensions": "^2.3.0",
         "diff": "^5.1.0",
-        "minimatch": "^9.0.0",
+        "minimatch": "^9.0.4",
         "npm-package-arg": "^11.0.1",
         "pacote": "^17.0.4",
-        "tar": "^6.2.0"
+        "tar": "^6.2.1"
       },
       "engines": {
         "node": "^16.14.0 || >=18.0.0"
       }
     },
     "node_modules/npm/node_modules/libnpmexec": {
-      "version": "7.0.8",
+      "version": "7.0.9",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
@@ -9543,7 +9519,7 @@
         "npmlog": "^7.0.1",
         "pacote": "^17.0.4",
         "proc-log": "^3.0.0",
-        "read": "^2.0.0",
+        "read": "^3.0.1",
         "read-package-json-fast": "^3.0.2",
         "semver": "^7.3.7",
         "walk-up-path": "^3.0.1"
@@ -9553,7 +9529,7 @@
       }
     },
     "node_modules/npm/node_modules/libnpmfund": {
-      "version": "5.0.5",
+      "version": "5.0.6",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
@@ -9564,31 +9540,31 @@
       }
     },
     "node_modules/npm/node_modules/libnpmhook": {
-      "version": "10.0.1",
+      "version": "10.0.2",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
         "aproba": "^2.0.0",
-        "npm-registry-fetch": "^16.0.0"
+        "npm-registry-fetch": "^16.2.0"
       },
       "engines": {
         "node": "^16.14.0 || >=18.0.0"
       }
     },
     "node_modules/npm/node_modules/libnpmorg": {
-      "version": "6.0.2",
+      "version": "6.0.3",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
         "aproba": "^2.0.0",
-        "npm-registry-fetch": "^16.0.0"
+        "npm-registry-fetch": "^16.2.0"
       },
       "engines": {
         "node": "^16.14.0 || >=18.0.0"
       }
     },
     "node_modules/npm/node_modules/libnpmpack": {
-      "version": "6.0.7",
+      "version": "6.0.8",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
@@ -9602,14 +9578,14 @@
       }
     },
     "node_modules/npm/node_modules/libnpmpublish": {
-      "version": "9.0.4",
+      "version": "9.0.5",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
         "ci-info": "^4.0.0",
         "normalize-package-data": "^6.0.0",
         "npm-package-arg": "^11.0.1",
-        "npm-registry-fetch": "^16.0.0",
+        "npm-registry-fetch": "^16.2.0",
         "proc-log": "^3.0.0",
         "semver": "^7.3.7",
         "sigstore": "^2.2.0",
@@ -9620,23 +9596,23 @@
       }
     },
     "node_modules/npm/node_modules/libnpmsearch": {
-      "version": "7.0.1",
+      "version": "7.0.2",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
-        "npm-registry-fetch": "^16.0.0"
+        "npm-registry-fetch": "^16.2.0"
       },
       "engines": {
         "node": "^16.14.0 || >=18.0.0"
       }
     },
     "node_modules/npm/node_modules/libnpmteam": {
-      "version": "6.0.1",
+      "version": "6.0.2",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
         "aproba": "^2.0.0",
-        "npm-registry-fetch": "^16.0.0"
+        "npm-registry-fetch": "^16.2.0"
       },
       "engines": {
         "node": "^16.14.0 || >=18.0.0"
@@ -9687,9 +9663,7 @@
       }
     },
     "node_modules/npm/node_modules/minimatch": {
-      "version": "9.0.3",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",
-      "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",
+      "version": "9.0.4",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
@@ -9704,8 +9678,6 @@
     },
     "node_modules/npm/node_modules/minipass": {
       "version": "7.0.4",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
-      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "inBundle": true,
       "license": "ISC",
       "engines": {
@@ -9881,7 +9853,7 @@
       }
     },
     "node_modules/npm/node_modules/node-gyp": {
-      "version": "10.0.1",
+      "version": "10.1.0",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -10021,10 +9993,11 @@
       }
     },
     "node_modules/npm/node_modules/npm-registry-fetch": {
-      "version": "16.1.0",
+      "version": "16.2.0",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
+        "@npmcli/redact": "^1.1.0",
         "make-fetch-happen": "^13.0.0",
         "minipass": "^7.0.2",
         "minipass-fetch": "^3.0.0",
@@ -10119,8 +10092,6 @@
     },
     "node_modules/npm/node_modules/path-key": {
       "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
-      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
       "inBundle": true,
       "license": "MIT",
       "engines": {
@@ -10128,13 +10099,11 @@
       }
     },
     "node_modules/npm/node_modules/path-scurry": {
-      "version": "1.10.1",
-      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz",
-      "integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==",
+      "version": "1.10.2",
       "inBundle": true,
       "license": "BlueOak-1.0.0",
       "dependencies": {
-        "lru-cache": "^9.1.1 || ^10.0.0",
+        "lru-cache": "^10.2.0",
         "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
       },
       "engines": {
@@ -10198,11 +10167,11 @@
       }
     },
     "node_modules/npm/node_modules/promzard": {
-      "version": "1.0.0",
+      "version": "1.0.1",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
-        "read": "^2.0.0"
+        "read": "^3.0.1"
       },
       "engines": {
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
@@ -10216,11 +10185,11 @@
       }
     },
     "node_modules/npm/node_modules/read": {
-      "version": "2.1.0",
+      "version": "3.0.1",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
-        "mute-stream": "~1.0.0"
+        "mute-stream": "^1.0.0"
       },
       "engines": {
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
@@ -10270,8 +10239,6 @@
     },
     "node_modules/npm/node_modules/safer-buffer": {
       "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
-      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
       "inBundle": true,
       "license": "MIT",
       "optional": true
@@ -10308,8 +10275,6 @@
     },
     "node_modules/npm/node_modules/shebang-command": {
       "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
-      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -10321,8 +10286,6 @@
     },
     "node_modules/npm/node_modules/shebang-regex": {
       "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
-      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
       "inBundle": true,
       "license": "MIT",
       "engines": {
@@ -10331,8 +10294,6 @@
     },
     "node_modules/npm/node_modules/signal-exit": {
       "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
-      "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
       "inBundle": true,
       "license": "ISC",
       "engines": {
@@ -10434,8 +10395,6 @@
     },
     "node_modules/npm/node_modules/string-width": {
       "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -10450,8 +10409,6 @@
     "node_modules/npm/node_modules/string-width-cjs": {
       "name": "string-width",
       "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -10465,8 +10422,6 @@
     },
     "node_modules/npm/node_modules/strip-ansi": {
       "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -10479,8 +10434,6 @@
     "node_modules/npm/node_modules/strip-ansi-cjs": {
       "name": "strip-ansi",
       "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -10502,7 +10455,7 @@
       }
     },
     "node_modules/npm/node_modules/tar": {
-      "version": "6.2.0",
+      "version": "6.2.1",
       "inBundle": true,
       "license": "ISC",
       "dependencies": {
@@ -10602,8 +10555,6 @@
     },
     "node_modules/npm/node_modules/util-deprecate": {
       "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
-      "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
       "inBundle": true,
       "license": "MIT"
     },
@@ -10672,8 +10623,6 @@
     },
     "node_modules/npm/node_modules/wrap-ansi": {
       "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
-      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -10691,8 +10640,6 @@
     "node_modules/npm/node_modules/wrap-ansi-cjs": {
       "name": "wrap-ansi",
       "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
-      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -10723,8 +10670,6 @@
     },
     "node_modules/npm/node_modules/wrap-ansi/node_modules/ansi-regex": {
       "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
-      "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
       "inBundle": true,
       "license": "MIT",
       "engines": {
@@ -10736,8 +10681,6 @@
     },
     "node_modules/npm/node_modules/wrap-ansi/node_modules/emoji-regex": {
       "version": "9.2.2",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
-      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==",
       "inBundle": true,
       "license": "MIT"
     },
@@ -10759,8 +10702,6 @@
     },
     "node_modules/npm/node_modules/wrap-ansi/node_modules/strip-ansi": {
       "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
-      "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
       "inBundle": true,
       "license": "MIT",
       "dependencies": {

docs/pages/Developing/API-docs.md

@@ -227,3 +227,124 @@ JSON response indicating the status of the operation:
 ```json
 { "status": "ok" }
 ```
+
+### 7. /api/get_api_keys
+**Description:**
+
+The endpoint retrieves a list of API keys for the user.
+
+**Request:**
+
+**Method**: `GET`
+
+**Sample JavaScript Fetch Request:**
+```js
+// get_api_keys (GET http://127.0.0.1:5000/api/get_api_keys)
+fetch("http://localhost:5001/api/get_api_keys", {
+      "method": "GET",
+      "headers": {
+            "Content-Type": "application/json; charset=utf-8"
+      },
+})
+.then((res) => res.text())
+.then(console.log.bind(console))
+
+```
+**Response:**
+
+JSON response with a list of created API keys:
+
+```json
+[
+      {
+        "id": "string",
+        "name": "string",
+        "key": "string",
+        "source": "string"
+      },
+      ...
+    ]
+```
+
+### 8. /api/create_api_key
+
+**Description:**
+
+Create a new API key for the user.
+
+**Request:**
+
+**Method**: `POST`
+
+**Headers**: Content-Type should be set to `application/json; charset=utf-8`
+
+**Request Body**: JSON object with the following fields:
+* `name` — A name for the API key.
+* `source` — The source documents that will be used.
+* `prompt_id` — The prompt ID.
+* `chunks` — The number of chunks used to process an answer.
+
+Here is a JavaScript Fetch Request example:
+```js
+// create_api_key (POST http://127.0.0.1:5000/api/create_api_key)
+fetch("http://127.0.0.1:5000/api/create_api_key", {
+      "method": "POST",
+      "headers": {
+            "Content-Type": "application/json; charset=utf-8"
+      },
+      "body": JSON.stringify({"name":"Example Key Name",
+          "source":"Example Source",
+          "prompt_id":"creative",
+          "chunks":"2"})
+})
+.then((res) => res.json())
+.then(console.log.bind(console))
+```
+
+**Response**
+
+In response, you will get a JSON document containing the `id` and `key`:
+```json
+{
+  "id": "string",
+  "key": "string"
+}
+```
+
+### 9. /api/delete_api_key
+
+**Description:**
+
+Delete an API key for the user.
+
+**Request:**
+
+**Method**: `POST`
+
+**Headers**: Content-Type should be set to `application/json; charset=utf-8`
+
+**Request Body**: JSON object with the field:
+* `id` — The unique identifier of the API key to be deleted.
+
+Here is a JavaScript Fetch Request example:
+```js
+// delete_api_key (POST http://127.0.0.1:5000/api/delete_api_key)
+fetch("http://127.0.0.1:5000/api/delete_api_key", {
+      "method": "POST",
+      "headers": {
+            "Content-Type": "application/json; charset=utf-8"
+      },
+      "body": JSON.stringify({"id":"API_KEY_ID"})
+})
+.then((res) => res.json())
+.then(console.log.bind(console))
+```
+
+**Response:**
+
+In response, you will get a JSON document indicating the status of the operation:
+```json
+{
+  "status": "ok"
+}
+```

docs/pages/Extensions/_meta.json

@@ -4,7 +4,11 @@
     "href": "/Extensions/Chatwoot-extension"
   },
   "react-widget": {
-      "title": "🏗️ Widget setup",
-      "href": "/Extensions/react-widget"
-    }
+    "title": "🏗️ Widget setup",
+    "href": "/Extensions/react-widget"
+  },
+  "api-key-guide": {
+    "title": "🔐 API Keys guide",
+    "href": "/Extensions/api-key-guide"
+  }
 }

docs/pages/Extensions/api-key-guide.md

@@ -0,0 +1,30 @@
+## Guide to DocsGPT API Keys
+
+DocsGPT API keys are essential for developers and users who wish to integrate the  DocsGPT models into external applications, such as the our widget. This guide will walk you through the steps of obtaining an API key, starting from uploading your document to understanding the key variables associated with API keys.
+
+### Uploading Your Document
+
+Before creating your first API key, you must upload the document that will be linked to this key. You can upload your document through two methods:
+
+- **GUI Web App Upload:** A user-friendly graphical interface that allows for easy upload and management of documents.
+- **Using `/api/upload` Method:** For users comfortable with API calls, this method provides a direct way to upload documents.
+
+### Obtaining Your API Key
+
+After uploading your document, you can obtain an API key either through the graphical user interface or via an API call:
+
+- **Graphical User Interface:** Navigate to the Settings section of the DocsGPT web app, find the API Keys option, and press 'Create New' to generate your key.
+- **API Call:** Alternatively, you can use the `/api/create_api_key` endpoint to create a new API key. For detailed instructions, visit [DocsGPT API Documentation](https://docs.docsgpt.co.uk/Developing/API-docs#8-apicreate_api_key).
+
+### Understanding Key Variables
+
+Upon creating your API key, you will encounter several key variables. Each serves a specific purpose:
+
+- **Name:** Assign a name to your API key for easy identification.
+- **Source:** Indicates the source document(s) linked to your API key, which DocsGPT will use to generate responses.
+- **ID:** A unique identifier for your API key. You can view this by making a call to `/api/get_api_keys`.
+- **Key:** The API key itself, which will be used in your application to authenticate API requests.
+
+With your API key ready, you can now integrate DocsGPT into your application, such as the DocsGPT Widget or any other software, via `/api/answer` or `/stream` endpoints. The source document is preset with the API key, allowing you to bypass fields like `selectDocs` and `active_docs` during implementation.
+
+Congratulations on taking the first step towards enhancing your applications with DocsGPT! With this guide, you're now equipped to navigate the process of obtaining and understanding DocsGPT API keys.

docs/pages/_app.mdx

@@ -4,7 +4,7 @@ export default function MyApp({ Component, pageProps }) {
   return (
     <>
       <Component {...pageProps} />
-        <DocsGPTWidget selectDocs="local/docsgpt-sep.zip/"/>
+        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" />
     </>
   )
 }

extensions/react-widget/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "docsgpt",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "docsgpt",
-      "version": "0.3.6",
+      "version": "0.3.7",
       "license": "Apache-2.0",
       "dependencies": {
         "@babel/plugin-transform-flow-strip-types": "^7.23.3",

extensions/react-widget/package.json

@@ -19,7 +19,7 @@
   },
   "scripts": {
     "build": "parcel build src/index.ts",
-    "dev": "parcel",
+    "dev": "parcel src/index.html -p 3000",
     "test": "jest",
     "lint": "eslint",
     "check": "tsc --noEmit",

extensions/react-widget/src/components/DocsGPTWidget.tsx

@@ -1,8 +1,7 @@
 "use client";
 import { Fragment, useEffect, useRef, useState } from 'react'
 import { PaperPlaneIcon, RocketIcon, ExclamationTriangleIcon, Cross2Icon } from '@radix-ui/react-icons';
-import { MESSAGE_TYPE } from '../models/types';
-import { Query, Status } from '../models/types';
+import { MESSAGE_TYPE, Query, Status } from '../types/index';
 import MessageIcon from '../assets/message.svg'
 import { fetchAnswerStreaming } from '../requests/streamingApi';
 import styled, { keyframes, createGlobalStyle } from 'styled-components';
@@ -285,7 +284,7 @@ const Hero = ({ title, description }: { title: string, description: string }) =>
 export const DocsGPTWidget = ({
   apiHost = 'https://gptcloud.arc53.com',
   selectDocs = 'default',
-  apiKey = 'docsgpt-public',
+  apiKey = '82962c9a-aa77-4152-94e5-a4f84fd44c6a',
   avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
   title = 'Get AI assistance',
   description = 'DocsGPT\'s AI Chatbot is here to help',

extensions/react-widget/src/types/index.ts

@@ -0,0 +1,13 @@
+export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER' | 'ERROR';
+export type Status = 'idle' | 'loading' | 'failed';
+export type FEEDBACK = 'LIKE' | 'DISLIKE';
+
+export interface Query {
+  prompt: string;
+  response?: string;
+  feedback?: FEEDBACK;
+  error?: string;
+  sources?: { title: string; text: string }[];
+  conversationId?: string | null;
+  title?: string | null;
+}

frontend/package-lock.json

@@ -44,7 +44,7 @@
         "prettier-plugin-tailwindcss": "^0.2.2",
         "tailwindcss": "^3.2.4",
         "typescript": "^4.9.5",
-        "vite": "^5.0.12",
+        "vite": "^5.0.13",
         "vite-plugin-svgr": "^4.2.0"
       }
     },
@@ -7855,9 +7855,9 @@
       }
     },
     "node_modules/vite": {
-      "version": "5.0.12",
-      "resolved": "https://registry.npmjs.org/vite/-/vite-5.0.12.tgz",
-      "integrity": "sha512-4hsnEkG3q0N4Tzf1+t6NdN9dg/L3BM+q8SWgbSPnJvrgH2kgdyzfVJwbR1ic69/4uMJJ/3dqDZZE5/WwqW8U1w==",
+      "version": "5.0.13",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-5.0.13.tgz",
+      "integrity": "sha512-/9ovhv2M2dGTuA+dY93B9trfyWMDRQw2jdVBhHNP6wr0oF34wG2i/N55801iZIpgUpnHDm4F/FabGQLyc+eOgg==",
       "dev": true,
       "dependencies": {
         "esbuild": "^0.19.3",

frontend/package.json

@@ -55,7 +55,7 @@
     "prettier-plugin-tailwindcss": "^0.2.2",
     "tailwindcss": "^3.2.4",
     "typescript": "^4.9.5",
-    "vite": "^5.0.12",
+    "vite": "^5.0.13",
     "vite-plugin-svgr": "^4.2.0"
   }
 }

frontend/src/App.tsx

@@ -6,7 +6,7 @@ import PageNotFound from './PageNotFound';
 import { inject } from '@vercel/analytics';
 import { useMediaQuery } from './hooks';
 import { useState } from 'react';
-import Setting from './Setting';
+import Setting from './settings';
 
 inject();
 

frontend/src/Hero.tsx

@@ -7,9 +7,7 @@ export default function Hero({ className = '' }: { className?: string }) {
   const [isDarkTheme] = useDarkTheme();
   return (
     <div
-      className={`mt-14 ${
-        isMobile ? 'mb-2' : 'mb-12'
-      } flex flex-col text-black-1000 dark:text-bright-gray`}
+      className={`mt-14 mb-32 flex flex-col text-black-1000 dark:text-bright-gray lg:mt-6`}
     >
       <div className=" mb-2 flex items-center justify-center sm:mb-10">
         <p className="mr-2 text-4xl font-semibold">DocsGPT</p>
@@ -39,16 +37,14 @@ export default function Hero({ className = '' }: { className?: string }) {
         </>
       )}
       <div
-        className={`sections ${
-          isMobile ? '' : 'mt-1'
-        } flex flex-wrap items-center justify-center gap-2 sm:gap-1 md:gap-0`}
+        className={`mt-0 flex flex-wrap items-center justify-center gap-2 sm:mt-1 sm:gap-4 md:gap-4 lg:gap-0`}
       >
         {/* first */}
-        <div className="h-auto rounded-[50px] bg-gradient-to-l from-[#6EE7B7]/70  via-[#3B82F6] to-[#9333EA]/50 p-1 dark:from-[#D16FF8] dark:via-[#48E6E0] dark:to-[#C85EF6] md:h-60  md:rounded-tr-none md:rounded-br-none">
+        <div className="h-auto rounded-[50px] bg-gradient-to-l from-[#6EE7B7]/70  via-[#3B82F6] to-[#9333EA]/50 p-1 dark:from-[#D16FF8] dark:via-[#48E6E0] dark:to-[#C85EF6] lg:h-60  lg:rounded-tr-none lg:rounded-br-none">
           <div
             className={`h-full rounded-[45px] bg-white dark:bg-dark-charcoal p-${
               isMobile ? '3.5' : '6 py-8'
-            }  md:rounded-tr-none md:rounded-br-none`}
+            }  lg:rounded-tr-none lg:rounded-br-none`}
           >
             {/* Add Mobile check here */}
             {isMobile ? (
@@ -93,11 +89,11 @@ export default function Hero({ className = '' }: { className?: string }) {
           </div>
         </div>
         {/* second */}
-        <div className="h-auto rounded-[50px] bg-gradient-to-r from-[#6EE7B7]/70 via-[#3B82F6] to-[#9333EA]/50 p-1 dark:from-[#D16FF8] dark:via-[#48E6E0] dark:to-[#C85EF6] md:h-60  md:rounded-none  md:py-1 md:px-0">
+        <div className="h-auto rounded-[50px] bg-gradient-to-r from-[#6EE7B7]/70 via-[#3B82F6] to-[#9333EA]/50 p-1 dark:from-[#D16FF8] dark:via-[#48E6E0] dark:to-[#C85EF6] lg:h-60  lg:rounded-none  lg:py-1 lg:px-0">
           <div
             className={`h-full rounded-[45px] bg-white dark:bg-dark-charcoal p-${
               isMobile ? '3.5' : '6 py-6'
-            }  md:rounded-none`}
+            }  lg:rounded-none`}
           >
             {/* Add Mobile check here */}
             {isMobile ? (
@@ -138,7 +134,7 @@ export default function Hero({ className = '' }: { className?: string }) {
           </div>
         </div>
         {/* third */}
-        <div className="h-auto rounded-[50px] bg-gradient-to-l from-[#6EE7B7]/70  via-[#3B82F6] to-[#9333EA]/50 p-1 dark:from-[#D16FF8] dark:via-[#48E6E0] dark:to-[#C85EF6] md:h-60 md:rounded-tl-none md:rounded-bl-none ">
+        <div className="h-auto rounded-[50px] bg-gradient-to-l from-[#6EE7B7]/70  via-[#3B82F6] to-[#9333EA]/50 p-1 dark:from-[#D16FF8] dark:via-[#48E6E0] dark:to-[#C85EF6] lg:h-60 lg:rounded-tl-none lg:rounded-bl-none ">
           <div
             className={`firefox h-full rounded-[45px] bg-white dark:bg-dark-charcoal p-${
               isMobile ? '3.5' : '6 px-6 '

frontend/src/Navigation.tsx

@@ -257,7 +257,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             New Chat
           </p>
         </NavLink>
-        <div className="mb-auto h-[56vh] overflow-x-hidden overflow-y-scroll dark:text-white">
+        <div className="mb-auto h-[56vh] overflow-y-auto overflow-x-hidden dark:text-white">
           {conversations && (
             <div>
               <p className="ml-6 mt-3 text-sm font-semibold">Chats</p>

frontend/src/Setting.tsx

@@ -1,725 +0,0 @@
-import React, { useState, useEffect } from 'react';
-import { useSelector, useDispatch } from 'react-redux';
-import ArrowLeft from './assets/arrow-left.svg';
-import ArrowRight from './assets/arrow-right.svg';
-import Trash from './assets/trash.svg';
-import {
-  selectPrompt,
-  setPrompt,
-  selectSourceDocs,
-  setSourceDocs,
-} from './preferences/preferenceSlice';
-import { Doc } from './preferences/preferenceApi';
-import { useDarkTheme } from './hooks';
-import Dropdown from './components/Dropdown';
-type PromptProps = {
-  prompts: { name: string; id: string; type: string }[];
-  selectedPrompt: { name: string; id: string; type: string };
-  onSelectPrompt: (name: string, id: string, type: string) => void;
-  setPrompts: (prompts: { name: string; id: string; type: string }[]) => void;
-  apiHost: string;
-};
-
-const Setting: React.FC = () => {
-  const tabs = ['General', 'Prompts', 'Documents'];
-  //const tabs = ['General', 'Prompts', 'Documents', 'Widgets'];
-
-  const [activeTab, setActiveTab] = useState('General');
-  const [prompts, setPrompts] = useState<
-    { name: string; id: string; type: string }[]
-  >([]);
-  const selectedPrompt = useSelector(selectPrompt);
-  const [isAddPromptModalOpen, setAddPromptModalOpen] = useState(false);
-  const documents = useSelector(selectSourceDocs);
-  const [isAddDocumentModalOpen, setAddDocumentModalOpen] = useState(false);
-
-  const dispatch = useDispatch();
-
-  const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
-  const [widgetScreenshot, setWidgetScreenshot] = useState<File | null>(null);
-
-  const updateWidgetScreenshot = (screenshot: File | null) => {
-    setWidgetScreenshot(screenshot);
-  };
-
-  useEffect(() => {
-    const fetchPrompts = async () => {
-      try {
-        const response = await fetch(`${apiHost}/api/get_prompts`);
-        if (!response.ok) {
-          throw new Error('Failed to fetch prompts');
-        }
-        const promptsData = await response.json();
-        setPrompts(promptsData);
-      } catch (error) {
-        console.error(error);
-      }
-    };
-
-    fetchPrompts();
-  }, []);
-
-  const onDeletePrompt = (name: string, id: string) => {
-    setPrompts(prompts.filter((prompt) => prompt.id !== id));
-
-    fetch(`${apiHost}/api/delete_prompt`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      // send id in body only
-      body: JSON.stringify({ id: id }),
-    })
-      .then((response) => {
-        if (!response.ok) {
-          throw new Error('Failed to delete prompt');
-        }
-      })
-      .catch((error) => {
-        console.error(error);
-      });
-  };
-
-  const handleDeleteClick = (index: number, doc: Doc) => {
-    const docPath = 'indexes/' + 'local' + '/' + doc.name;
-
-    fetch(`${apiHost}/api/delete_old?path=${docPath}`, {
-      method: 'GET',
-    })
-      .then((response) => {
-        if (response.ok && documents) {
-          const updatedDocuments = [
-            ...documents.slice(0, index),
-            ...documents.slice(index + 1),
-          ];
-          dispatch(setSourceDocs(updatedDocuments));
-        }
-      })
-      .catch((error) => console.error(error));
-  };
-
-  return (
-    <div className="wa p-4 pt-20 md:p-12">
-      <p className="text-2xl font-bold text-eerie-black dark:text-bright-gray">
-        Settings
-      </p>
-      <div className="mt-6 flex flex-row items-center space-x-4 overflow-x-auto md:space-x-8 ">
-        <div className="md:hidden">
-          <button
-            onClick={() => scrollTabs(-1)}
-            className="flex h-8 w-8 items-center justify-center rounded-full border-2 border-purple-30 transition-all hover:bg-gray-100"
-          >
-            <img src={ArrowLeft} alt="left-arrow" className="h-6 w-6" />
-          </button>
-        </div>
-        <div className="flex flex-nowrap space-x-4 overflow-x-auto md:space-x-8">
-          {tabs.map((tab, index) => (
-            <button
-              key={index}
-              onClick={() => setActiveTab(tab)}
-              className={`h-9 rounded-3xl px-4 font-bold ${
-                activeTab === tab
-                  ? 'bg-purple-3000 text-purple-30 dark:bg-dark-charcoal'
-                  : 'text-gray-6000'
-              }`}
-            >
-              {tab}
-            </button>
-          ))}
-        </div>
-        <div className="md:hidden">
-          <button
-            onClick={() => scrollTabs(1)}
-            className="flex h-8 w-8 items-center justify-center rounded-full border-2 border-purple-30 hover:bg-gray-100"
-          >
-            <img src={ArrowRight} alt="right-arrow" className="h-6 w-6" />
-          </button>
-        </div>
-      </div>
-      {renderActiveTab()}
-
-      {/* {activeTab === 'Widgets' && (
-        <Widgets
-          widgetScreenshot={widgetScreenshot}
-          onWidgetScreenshotChange={updateWidgetScreenshot}
-        />
-      )} */}
-    </div>
-  );
-
-  function scrollTabs(direction: number) {
-    const container = document.querySelector('.flex-nowrap');
-    if (container) {
-      container.scrollLeft += direction * 100; // Adjust the scroll amount as needed
-    }
-  }
-
-  function renderActiveTab() {
-    switch (activeTab) {
-      case 'General':
-        return <General />;
-      case 'Prompts':
-        return (
-          <Prompts
-            prompts={prompts}
-            selectedPrompt={selectedPrompt}
-            onSelectPrompt={(name, id, type) =>
-              dispatch(setPrompt({ name: name, id: id, type: type }))
-            }
-            setPrompts={setPrompts}
-            apiHost={apiHost}
-          />
-        );
-      case 'Documents':
-        return (
-          <Documents
-            documents={documents}
-            handleDeleteDocument={handleDeleteClick}
-          />
-        );
-      case 'Widgets':
-        return (
-          <Widgets
-            widgetScreenshot={widgetScreenshot} // Add this line
-            onWidgetScreenshotChange={updateWidgetScreenshot} // Add this line
-          />
-        );
-      default:
-        return null;
-    }
-  }
-};
-
-const General: React.FC = () => {
-  const themes = ['Light', 'Dark'];
-  const languages = ['English'];
-  const [isDarkTheme, toggleTheme] = useDarkTheme();
-  const [selectedTheme, setSelectedTheme] = useState(
-    isDarkTheme ? 'Dark' : 'Light',
-  );
-  const [selectedLanguage, setSelectedLanguage] = useState(languages[0]);
-  return (
-    <div className="mt-[59px]">
-      <div className="mb-4">
-        <p className="font-bold text-jet dark:text-bright-gray">Select Theme</p>
-        <Dropdown
-          options={themes}
-          selectedValue={selectedTheme}
-          onSelect={(option: string) => {
-            setSelectedTheme(option);
-            option !== selectedTheme && toggleTheme();
-          }}
-        />
-      </div>
-      <div>
-        <p className="font-bold text-jet dark:text-bright-gray">
-          Select Language
-        </p>
-        <Dropdown
-          options={languages}
-          selectedValue={selectedLanguage}
-          onSelect={setSelectedLanguage}
-        />
-      </div>
-    </div>
-  );
-};
-
-export default Setting;
-
-const Prompts: React.FC<PromptProps> = ({
-  prompts,
-  selectedPrompt,
-  onSelectPrompt,
-  setPrompts,
-  apiHost,
-}) => {
-  const handleSelectPrompt = ({
-    name,
-    id,
-    type,
-  }: {
-    name: string;
-    id: string;
-    type: string;
-  }) => {
-    setNewPromptName(name);
-    onSelectPrompt(name, id, type);
-  };
-  const [newPromptName, setNewPromptName] = useState(selectedPrompt.name);
-  const [newPromptContent, setNewPromptContent] = useState('');
-
-  const handleAddPrompt = async () => {
-    try {
-      const response = await fetch(`${apiHost}/api/create_prompt`, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          name: newPromptName,
-          content: newPromptContent,
-        }),
-      });
-      if (!response.ok) {
-        throw new Error('Failed to add prompt');
-      }
-      const newPrompt = await response.json();
-      if (setPrompts) {
-        setPrompts([
-          ...prompts,
-          { name: newPromptName, id: newPrompt.id, type: 'private' },
-        ]);
-      }
-      onSelectPrompt(newPromptName, newPrompt.id, newPromptContent);
-      setNewPromptName(newPromptName);
-    } catch (error) {
-      console.error(error);
-    }
-  };
-
-  const handleDeletePrompt = () => {
-    setPrompts(prompts.filter((prompt) => prompt.id !== selectedPrompt.id));
-    console.log('selectedPrompt.id', selectedPrompt.id);
-
-    fetch(`${apiHost}/api/delete_prompt`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({ id: selectedPrompt.id }),
-    })
-      .then((response) => {
-        if (!response.ok) {
-          throw new Error('Failed to delete prompt');
-        }
-        // get 1st prompt and set it as selected
-        if (prompts.length > 0) {
-          onSelectPrompt(prompts[0].name, prompts[0].id, prompts[0].type);
-          setNewPromptName(prompts[0].name);
-        }
-      })
-      .catch((error) => {
-        console.error(error);
-      });
-  };
-
-  useEffect(() => {
-    const fetchPromptContent = async () => {
-      console.log('fetching prompt content');
-      try {
-        const response = await fetch(
-          `${apiHost}/api/get_single_prompt?id=${selectedPrompt.id}`,
-          {
-            method: 'GET',
-            headers: {
-              'Content-Type': 'application/json',
-            },
-          },
-        );
-        if (!response.ok) {
-          throw new Error('Failed to fetch prompt content');
-        }
-        const promptContent = await response.json();
-        setNewPromptContent(promptContent.content);
-      } catch (error) {
-        console.error(error);
-      }
-    };
-
-    fetchPromptContent();
-  }, [selectedPrompt]);
-
-  const handleSaveChanges = () => {
-    fetch(`${apiHost}/api/update_prompt`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        id: selectedPrompt.id,
-        name: newPromptName,
-        content: newPromptContent,
-      }),
-    })
-      .then((response) => {
-        if (!response.ok) {
-          throw new Error('Failed to update prompt');
-        }
-        onSelectPrompt(newPromptName, selectedPrompt.id, selectedPrompt.type);
-        setNewPromptName(newPromptName);
-      })
-      .catch((error) => {
-        console.error(error);
-      });
-  };
-
-  return (
-    <div className="mt-[59px]">
-      <div className="mb-4">
-        <p className="font-semibold dark:text-bright-gray">Active Prompt</p>
-        <Dropdown
-          options={prompts}
-          selectedValue={selectedPrompt.name}
-          onSelect={handleSelectPrompt}
-        />
-      </div>
-
-      <div className="mb-4">
-        <p className="dark:text-bright-gray">Prompt name </p>{' '}
-        <p className="mb-2 text-xs italic text-eerie-black dark:text-bright-gray">
-          start by editing name
-        </p>
-        <input
-          type="text"
-          value={newPromptName}
-          placeholder="Active Prompt Name"
-          className="w-full rounded-lg border-2 p-2 dark:border-chinese-silver dark:bg-transparent dark:text-white"
-          onChange={(e) => setNewPromptName(e.target.value)}
-        />
-      </div>
-
-      <div className="mb-4">
-        <p className="mb-2 dark:text-bright-gray">Prompt content</p>
-        <textarea
-          className="h-32 w-full rounded-lg  border-2 p-2 dark:border-chinese-silver dark:bg-transparent dark:text-white"
-          value={newPromptContent}
-          onChange={(e) => setNewPromptContent(e.target.value)}
-          placeholder="Active prompt contents"
-        />
-      </div>
-
-      <div className="flex justify-between">
-        <button
-          className={`rounded-lg bg-green-500 px-4 py-2 font-bold text-white transition-all hover:bg-green-700 ${
-            newPromptName === selectedPrompt.name
-              ? 'cursor-not-allowed opacity-50'
-              : ''
-          }`}
-          onClick={handleAddPrompt}
-          disabled={newPromptName === selectedPrompt.name}
-        >
-          Add New Prompt
-        </button>
-        <button
-          className={`rounded-lg bg-red-500 px-4 py-2 font-bold text-white transition-all hover:bg-red-700 ${
-            selectedPrompt.type === 'public'
-              ? 'cursor-not-allowed opacity-50'
-              : ''
-          }`}
-          onClick={handleDeletePrompt}
-          disabled={selectedPrompt.type === 'public'}
-        >
-          Delete Prompt
-        </button>
-        <button
-          className={`rounded-lg bg-blue-500 px-4 py-2 font-bold text-white transition-all hover:bg-blue-700 ${
-            selectedPrompt.type === 'public'
-              ? 'cursor-not-allowed opacity-50'
-              : ''
-          }`}
-          onClick={handleSaveChanges}
-          disabled={selectedPrompt.type === 'public'}
-        >
-          Save Changes
-        </button>
-      </div>
-    </div>
-  );
-};
-
-type AddPromptModalProps = {
-  newPromptName: string;
-  onNewPromptNameChange: (name: string) => void;
-  onAddPrompt: () => void;
-  onClose: () => void;
-};
-
-const AddPromptModal: React.FC<AddPromptModalProps> = ({
-  newPromptName,
-  onNewPromptNameChange,
-  onAddPrompt,
-  onClose,
-}) => {
-  return (
-    <div className="fixed top-0 left-0 flex h-screen w-screen items-center justify-center bg-gray-900 bg-opacity-50">
-      <div className="rounded-3xl bg-white p-4">
-        <p className="mb-2 text-2xl font-bold text-jet">Add New Prompt</p>
-        <input
-          type="text"
-          placeholder="Enter Prompt Name"
-          value={newPromptName}
-          onChange={(e) => onNewPromptNameChange(e.target.value)}
-          className="mb-4 w-full rounded-3xl border-2 p-2 dark:border-chinese-silver"
-        />
-        <button
-          onClick={onAddPrompt}
-          className="rounded-3xl bg-purple-300 px-4 py-2 font-bold text-white transition-all hover:bg-purple-600"
-        >
-          Save
-        </button>
-        <button
-          onClick={onClose}
-          className="mt-4 rounded-3xl px-4 py-2 font-bold text-red-500"
-        >
-          Cancel
-        </button>
-      </div>
-    </div>
-  );
-};
-
-type DocumentsProps = {
-  documents: Doc[] | null;
-  handleDeleteDocument: (index: number, document: Doc) => void;
-};
-
-const Documents: React.FC<DocumentsProps> = ({
-  documents,
-  handleDeleteDocument,
-}) => {
-  return (
-    <div className="mt-8">
-      <div className="flex flex-col">
-        {/* <h2 className="text-xl font-semibold">Documents</h2> */}
-
-        <div className="mt-[27px] w-max overflow-x-auto rounded-xl border dark:border-chinese-silver">
-          <table className="block w-full table-auto content-center justify-center text-center dark:text-bright-gray">
-            <thead>
-              <tr>
-                <th className="border-r p-4 md:w-[244px]">Document Name</th>
-                <th className="w-[244px] border-r px-4 py-2">Vector Date</th>
-                <th className="w-[244px] border-r px-4 py-2">Type</th>
-                <th className="px-4 py-2"></th>
-              </tr>
-            </thead>
-            <tbody>
-              {documents &&
-                documents.map((document, index) => (
-                  <tr key={index}>
-                    <td className="border-r border-t px-4 py-2">
-                      {document.name}
-                    </td>
-                    <td className="border-r border-t px-4 py-2">
-                      {document.date}
-                    </td>
-                    <td className="border-r border-t px-4 py-2">
-                      {document.location === 'remote'
-                        ? 'Pre-loaded'
-                        : 'Private'}
-                    </td>
-                    <td className="border-t px-4 py-2">
-                      {document.location !== 'remote' && (
-                        <img
-                          src={Trash}
-                          alt="Delete"
-                          className="h-4 w-4 cursor-pointer hover:opacity-50"
-                          id={`img-${index}`}
-                          onClick={(event) => {
-                            event.stopPropagation();
-                            handleDeleteDocument(index, document);
-                          }}
-                        />
-                      )}
-                    </td>
-                  </tr>
-                ))}
-            </tbody>
-          </table>
-        </div>
-        {/* <button
-          onClick={toggleAddDocumentModal}
-          className="mt-10 w-32 rounded-lg bg-purple-300 px-4 py-2 font-bold text-white transition-all hover:bg-purple-600"
-        >
-          Add New
-        </button> */}
-      </div>
-
-      {/* {isAddDocumentModalOpen && (
-        <AddDocumentModal
-          newDocument={newDocument}
-          onNewDocumentChange={setNewDocument}
-          onAddDocument={addDocument}
-          onClose={toggleAddDocumentModal}
-        />
-      )} */}
-    </div>
-  );
-};
-
-type Document = {
-  name: string;
-  vectorDate: string;
-  vectorLocation: string;
-};
-
-// Modal for adding a new document
-type AddDocumentModalProps = {
-  newDocument: Document;
-  onNewDocumentChange: (document: Document) => void;
-  onAddDocument: () => void;
-  onClose: () => void;
-};
-
-const AddDocumentModal: React.FC<AddDocumentModalProps> = ({
-  newDocument,
-  onNewDocumentChange,
-  onAddDocument,
-  onClose,
-}) => {
-  return (
-    <div className="fixed top-0 left-0 flex h-screen w-screen items-center justify-center bg-gray-900 bg-opacity-50">
-      <div className="w-[50%] rounded-lg bg-white p-4">
-        <p className="mb-2 text-2xl font-bold text-jet">Add New Document</p>
-        <input
-          type="text"
-          placeholder="Document Name"
-          value={newDocument.name}
-          onChange={(e) =>
-            onNewDocumentChange({ ...newDocument, name: e.target.value })
-          }
-          className="mb-4 w-full rounded-lg border-2 p-2"
-        />
-        <input
-          type="text"
-          placeholder="Vector Date"
-          value={newDocument.vectorDate}
-          onChange={(e) =>
-            onNewDocumentChange({ ...newDocument, vectorDate: e.target.value })
-          }
-          className="mb-4 w-full rounded-lg border-2 p-2"
-        />
-        <input
-          type="text"
-          placeholder="Vector Location"
-          value={newDocument.vectorLocation}
-          onChange={(e) =>
-            onNewDocumentChange({
-              ...newDocument,
-              vectorLocation: e.target.value,
-            })
-          }
-          className="mb-4 w-full rounded-lg border-2 p-2"
-        />
-        <button
-          onClick={onAddDocument}
-          className="rounded-lg bg-purple-300 px-4 py-2 font-bold text-white transition-all hover:bg-purple-600"
-        >
-          Save
-        </button>
-        <button
-          onClick={onClose}
-          className="mt-4 rounded-lg px-4 py-2 font-bold text-red-500"
-        >
-          Cancel
-        </button>
-      </div>
-    </div>
-  );
-};
-
-const Widgets: React.FC<{
-  widgetScreenshot: File | null;
-  onWidgetScreenshotChange: (screenshot: File | null) => void;
-}> = ({ widgetScreenshot, onWidgetScreenshotChange }) => {
-  const widgetSources = ['Source 1', 'Source 2', 'Source 3'];
-  const widgetMethods = ['Method 1', 'Method 2', 'Method 3'];
-  const widgetTypes = ['Type 1', 'Type 2', 'Type 3'];
-
-  const [selectedWidgetSource, setSelectedWidgetSource] = useState(
-    widgetSources[0],
-  );
-  const [selectedWidgetMethod, setSelectedWidgetMethod] = useState(
-    widgetMethods[0],
-  );
-  const [selectedWidgetType, setSelectedWidgetType] = useState(widgetTypes[0]);
-
-  // const [widgetScreenshot, setWidgetScreenshot] = useState<File | null>(null);
-  const [widgetCode, setWidgetCode] = useState<string>(''); // Your widget code state
-
-  const handleScreenshotChange = (
-    event: React.ChangeEvent<HTMLInputElement>,
-  ) => {
-    const files = event.target.files;
-
-    if (files && files.length > 0) {
-      const selectedScreenshot = files[0];
-      onWidgetScreenshotChange(selectedScreenshot); // Update the screenshot in the parent component
-    }
-  };
-
-  const handleCopyToClipboard = () => {
-    // Create a new textarea element to select the text
-    const textArea = document.createElement('textarea');
-    textArea.value = widgetCode;
-    document.body.appendChild(textArea);
-
-    // Select and copy the text
-    textArea.select();
-    document.execCommand('copy');
-
-    // Clean up the textarea element
-    document.body.removeChild(textArea);
-  };
-
-  return (
-    <div>
-      <div className="mt-[59px]">
-        <p className="font-bold text-jet">Widget Source</p>
-        <Dropdown
-          options={widgetSources}
-          selectedValue={selectedWidgetSource}
-          onSelect={setSelectedWidgetSource}
-        />
-      </div>
-      <div className="mt-5">
-        <p className="font-bold text-jet">Widget Method</p>
-        <Dropdown
-          options={widgetMethods}
-          selectedValue={selectedWidgetMethod}
-          onSelect={setSelectedWidgetMethod}
-        />
-      </div>
-      <div className="mt-5">
-        <p className="font-bold text-jet">Widget Type</p>
-        <Dropdown
-          options={widgetTypes}
-          selectedValue={selectedWidgetType}
-          onSelect={setSelectedWidgetType}
-        />
-      </div>
-      <div className="mt-6">
-        <p className="font-bold text-jet">Widget Code Snippet</p>
-        <textarea
-          rows={4}
-          value={widgetCode}
-          onChange={(e) => setWidgetCode(e.target.value)}
-          className="mt-3 w-full rounded-lg border-2 p-2"
-        />
-      </div>
-      <div className="mt-1">
-        <button
-          onClick={handleCopyToClipboard}
-          className="rounded-lg bg-blue-400 px-2 py-2 font-bold text-white transition-all hover:bg-blue-600"
-        >
-          Copy
-        </button>
-      </div>
-
-      <div className="mt-4">
-        <p className="text-lg font-semibold">Widget Screenshot</p>
-        <input type="file" accept="image/*" onChange={handleScreenshotChange} />
-      </div>
-
-      {widgetScreenshot && (
-        <div className="mt-4">
-          <img
-            src={URL.createObjectURL(widgetScreenshot)}
-            alt="Widget Screenshot"
-            className="max-w-full rounded-lg border border-gray-300"
-          />
-        </div>
-      )}
-    </div>
-  );
-};

frontend/src/components/Dropdown.tsx

@@ -1,57 +1,68 @@
-import { useState } from 'react';
+import React from 'react';
 import Arrow2 from '../assets/dropdown-arrow.svg';
+import Edit from '../assets/edit.svg';
+import Trash from '../assets/trash.svg';
 
 function Dropdown({
   options,
   selectedValue,
   onSelect,
+  size = 'w-32',
+  rounded = 'xl',
+  showEdit,
+  onEdit,
   showDelete,
   onDelete,
+  placeholder,
 }: {
   options:
     | string[]
     | { name: string; id: string; type: string }[]
     | { label: string; value: string }[];
-  selectedValue: string | { label: string; value: string };
+  selectedValue: string | { label: string; value: string } | null;
   onSelect:
     | ((value: string) => void)
     | ((value: { name: string; id: string; type: string }) => void)
     | ((value: { label: string; value: string }) => void);
+  size?: string;
+  rounded?: 'xl' | '3xl';
+  showEdit?: boolean;
+  onEdit?: (value: { name: string; id: string; type: string }) => void;
   showDelete?: boolean;
   onDelete?: (value: string) => void;
+  placeholder?: string;
 }) {
-  const [isOpen, setIsOpen] = useState(false);
+  const [isOpen, setIsOpen] = React.useState(false);
   return (
     <div
-      className={
+      className={[
         typeof selectedValue === 'string'
-          ? 'relative mt-2 w-32'
-          : 'relative w-full align-middle'
-      }
+          ? 'relative mt-2'
+          : 'relative align-middle',
+        size,
+      ].join(' ')}
     >
       <button
         onClick={() => setIsOpen(!isOpen)}
-        className={`flex w-full cursor-pointer items-center justify-between border-2 bg-white p-3 dark:border-chinese-silver dark:bg-transparent ${
-          isOpen
-            ? typeof selectedValue === 'string'
-              ? 'rounded-t-xl'
-              : 'rounded-t-2xl'
-            : typeof selectedValue === 'string'
-            ? 'rounded-xl'
-            : 'rounded-full'
+        className={`flex w-full cursor-pointer items-center justify-between border-2 border-silver bg-white px-5 py-3 dark:border-chinese-silver dark:bg-transparent ${
+          isOpen ? `rounded-t-${rounded}` : `rounded-${rounded}`
         }`}
       >
         {typeof selectedValue === 'string' ? (
-          <span className="flex-1 overflow-hidden text-ellipsis dark:text-bright-gray">
+          <span className="overflow-hidden text-ellipsis dark:text-bright-gray">
             {selectedValue}
           </span>
         ) : (
           <span
             className={`overflow-hidden text-ellipsis dark:text-bright-gray ${
-              !selectedValue && 'text-silver'
+              !selectedValue && 'text-silver dark:text-gray-400'
             }`}
           >
-            {selectedValue ? selectedValue.label : 'From URL'}
+            {selectedValue
+              ? selectedValue.label
+              : placeholder
+              ? placeholder
+              : 'From URL'}
           </span>
         )}
         <img
@@ -63,7 +74,7 @@ function Dropdown({
         />
       </button>
       {isOpen && (
-        <div className="absolute left-0 right-0 z-50 -mt-1 overflow-y-auto rounded-b-xl border-2 bg-white shadow-lg dark:border-chinese-silver dark:bg-dark-charcoal">
+        <div className="absolute left-0 right-0 z-20 -mt-1 max-h-40 overflow-y-auto rounded-b-xl border-2 border-silver bg-white shadow-lg dark:border-chinese-silver dark:bg-dark-charcoal">
           {options.map((option: any, index) => (
             <div
               key={index}
@@ -74,7 +85,7 @@ function Dropdown({
                   onSelect(option);
                   setIsOpen(false);
                 }}
-                className="ml-2 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3 dark:text-light-gray"
+                className="ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3 dark:text-light-gray"
               >
                 {typeof option === 'string'
                   ? option
@@ -82,9 +93,35 @@ function Dropdown({
                   ? option.name
                   : option.label}
               </span>
+              {showEdit && onEdit && (
+                <img
+                  src={Edit}
+                  alt="Edit"
+                  className="mr-4 h-4 w-4 cursor-pointer hover:opacity-50"
+                  onClick={() => {
+                    onEdit({
+                      id: option.id,
+                      name: option.name,
+                      type: option.type,
+                    });
+                    setIsOpen(false);
+                  }}
+                />
+              )}
               {showDelete && onDelete && (
-                <button onClick={() => onDelete(option)} className="p-2">
-                  Delete
+                <button
+                  onClick={() => onDelete(option.id)}
+                  disabled={option.type === 'public'}
+                >
+                  <img
+                    src={Trash}
+                    alt="Delete"
+                    className={`mr-2 h-4 w-4 cursor-pointer hover:opacity-50 ${
+                      option.type === 'public'
+                        ? 'cursor-not-allowed opacity-50'
+                        : ''
+                    }`}
+                  />
                 </button>
               )}
             </div>

frontend/src/components/SourceDropdown.tsx

@@ -24,6 +24,12 @@ function SourceDropdown({
   const embeddingsName =
     import.meta.env.VITE_EMBEDDINGS_NAME ||
     'huggingface_sentence-transformers/all-mpnet-base-v2';
+
+  const handleEmptyDocumentSelect = () => {
+    dispatch(setSelectedDocs(null));
+    setIsDocsListOpen(false);
+  };
+
   return (
     <div className="relative w-5/6 rounded-3xl">
       <button
@@ -35,7 +41,7 @@ function SourceDropdown({
         <span className="ml-1 mr-2 flex-1 overflow-hidden text-ellipsis text-left dark:text-bright-gray">
           <div className="flex flex-row gap-2">
             <p className="max-w-3/4 truncate whitespace-nowrap">
-              {selectedDocs?.name}
+              {selectedDocs?.name || ''}
             </p>
             <p className="flex flex-col items-center justify-center">
               {selectedDocs?.version}
@@ -93,6 +99,14 @@ function SourceDropdown({
               <p className="ml-5 py-3">No default documentation.</p>
             </div>
           )}
+          <div
+            className="flex cursor-pointer items-center justify-between hover:bg-gray-100 dark:text-bright-gray dark:hover:bg-purple-taupe"
+            onClick={handleEmptyDocumentSelect}
+          >
+            <span className="ml-4 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3">
+              Empty
+            </span>
+          </div>
         </div>
       )}
     </div>

frontend/src/conversation/ConversationBubble.tsx

@@ -160,7 +160,10 @@ const ConversationBubble = forwardRef<
             >
               {message}
             </ReactMarkdown>
-            {DisableSourceFE || type === 'ERROR' ? null : (
+            {DisableSourceFE ||
+            type === 'ERROR' ||
+            !sources ||
+            sources.length === 0 ? null : (
               <>
                 <span className="mt-3 h-px w-full bg-[#DEDEDE]"></span>
                 <div className="mt-3 flex w-full flex-row flex-wrap items-center justify-start gap-2">

frontend/src/conversation/conversationApi.ts

@@ -3,14 +3,41 @@ import { Doc } from '../preferences/preferenceApi';
 
 const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
 
+function getDocPath(selectedDocs: Doc | null): string {
+  let docPath = 'default';
+
+  if (selectedDocs) {
+    let namePath = selectedDocs.name;
+    if (selectedDocs.language === namePath) {
+      namePath = '.project';
+    }
+    if (selectedDocs.location === 'local') {
+      docPath = 'local' + '/' + selectedDocs.name + '/';
+    } else if (selectedDocs.location === 'remote') {
+      docPath =
+        selectedDocs.language +
+        '/' +
+        namePath +
+        '/' +
+        selectedDocs.version +
+        '/' +
+        selectedDocs.model +
+        '/';
+    } else if (selectedDocs.location === 'custom') {
+      docPath = selectedDocs.docLink;
+    }
+  }
+
+  return docPath;
+}
 export function fetchAnswerApi(
   question: string,
   signal: AbortSignal,
-  apiKey: string,
-  selectedDocs: Doc,
+  selectedDocs: Doc | null,
   history: Array<any> = [],
   conversationId: string | null,
   promptId: string | null,
+  chunks: string,
 ): Promise<
   | {
       result: any;
@@ -28,25 +55,7 @@ export function fetchAnswerApi(
       title: any;
     }
 > {
-  let namePath = selectedDocs.name;
-  if (selectedDocs.language === namePath) {
-    namePath = '.project';
-  }
-
-  let docPath = 'default';
-  if (selectedDocs.location === 'local') {
-    docPath = 'local' + '/' + selectedDocs.name + '/';
-  } else if (selectedDocs.location === 'remote') {
-    docPath =
-      selectedDocs.language +
-      '/' +
-      namePath +
-      '/' +
-      selectedDocs.version +
-      '/' +
-      selectedDocs.model +
-      '/';
-  }
+  const docPath = getDocPath(selectedDocs);
   //in history array remove all keys except prompt and response
   history = history.map((item) => {
     return { prompt: item.prompt, response: item.response };
@@ -59,12 +68,11 @@ export function fetchAnswerApi(
     },
     body: JSON.stringify({
       question: question,
-      api_key: apiKey,
-      embeddings_key: apiKey,
       history: history,
       active_docs: docPath,
       conversation_id: conversationId,
       prompt_id: promptId,
+      chunks: chunks,
     }),
     signal,
   })
@@ -90,32 +98,14 @@ export function fetchAnswerApi(
 export function fetchAnswerSteaming(
   question: string,
   signal: AbortSignal,
-  apiKey: string,
-  selectedDocs: Doc,
+  selectedDocs: Doc | null,
   history: Array<any> = [],
   conversationId: string | null,
   promptId: string | null,
+  chunks: string,
   onEvent: (event: MessageEvent) => void,
 ): Promise<Answer> {
-  let namePath = selectedDocs.name;
-  if (selectedDocs.language === namePath) {
-    namePath = '.project';
-  }
-
-  let docPath = 'default';
-  if (selectedDocs.location === 'local') {
-    docPath = 'local' + '/' + selectedDocs.name + '/';
-  } else if (selectedDocs.location === 'remote') {
-    docPath =
-      selectedDocs.language +
-      '/' +
-      namePath +
-      '/' +
-      selectedDocs.version +
-      '/' +
-      selectedDocs.model +
-      '/';
-  }
+  const docPath = getDocPath(selectedDocs);
 
   history = history.map((item) => {
     return { prompt: item.prompt, response: item.response };
@@ -124,12 +114,11 @@ export function fetchAnswerSteaming(
   return new Promise<Answer>((resolve, reject) => {
     const body = {
       question: question,
-      api_key: apiKey,
-      embeddings_key: apiKey,
       active_docs: docPath,
       history: JSON.stringify(history),
       conversation_id: conversationId,
       prompt_id: promptId,
+      chunks: chunks,
     };
     fetch(apiHost + '/stream', {
       method: 'POST',
@@ -188,41 +177,19 @@ export function fetchAnswerSteaming(
 }
 export function searchEndpoint(
   question: string,
-  apiKey: string,
-  selectedDocs: Doc,
+  selectedDocs: Doc | null,
   conversation_id: string | null,
   history: Array<any> = [],
+  chunks: string,
 ) {
-  /*
-  "active_docs": "default",
-  "question": "Summarise",
-  "conversation_id": null,
-  "history": "[]" */
-  let namePath = selectedDocs.name;
-  if (selectedDocs.language === namePath) {
-    namePath = '.project';
-  }
-
-  let docPath = 'default';
-  if (selectedDocs.location === 'local') {
-    docPath = 'local' + '/' + selectedDocs.name + '/';
-  } else if (selectedDocs.location === 'remote') {
-    docPath =
-      selectedDocs.language +
-      '/' +
-      namePath +
-      '/' +
-      selectedDocs.version +
-      '/' +
-      selectedDocs.model +
-      '/';
-  }
+  const docPath = getDocPath(selectedDocs);
 
   const body = {
     question: question,
     active_docs: docPath,
     conversation_id,
     history,
+    chunks: chunks,
   };
   return fetch(`${apiHost}/api/search`, {
     method: 'POST',

frontend/src/conversation/conversationSlice.ts

@@ -23,11 +23,11 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
         await fetchAnswerSteaming(
           question,
           signal,
-          state.preference.apiKey,
           state.preference.selectedDocs!,
           state.conversation.queries,
           state.conversation.conversationId,
           state.preference.prompt.id,
+          state.preference.chunks,
 
           (event) => {
             const data = JSON.parse(event.data);
@@ -47,10 +47,10 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
               searchEndpoint(
                 //search for sources post streaming
                 question,
-                state.preference.apiKey,
                 state.preference.selectedDocs!,
                 state.conversation.conversationId,
                 state.conversation.queries,
+                state.preference.chunks,
               ).then((sources) => {
                 //dispatch streaming sources
                 dispatch(
@@ -81,11 +81,11 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
         const answer = await fetchAnswerApi(
           question,
           signal,
-          state.preference.apiKey,
           state.preference.selectedDocs!,
           state.conversation.queries,
           state.conversation.conversationId,
           state.preference.prompt.id,
+          state.preference.chunks,
         );
         if (answer) {
           let sourcesPrepped = [];

frontend/src/index.css

@@ -2,6 +2,26 @@
 @tailwind components;
 @tailwind utilities;
 
+::-webkit-scrollbar {
+  width: 8px;
+}
+::-webkit-scrollbar-track {
+  background: #f1f1f1;
+}
+.dark ::-webkit-scrollbar-track {
+  background: #2f3036;
+}
+::-webkit-scrollbar-thumb {
+  background: #888;
+  border-radius: 40px;
+}
+::-webkit-scrollbar-thumb:hover {
+  background: #555;
+}
+.dark ::-webkit-scrollbar-thumb:hover {
+  background: #b1afaf;
+}
+
 /*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */
 
 /* Document

frontend/src/models/misc.ts

@@ -3,3 +3,42 @@ export type ActiveState = 'ACTIVE' | 'INACTIVE';
 export type User = {
   avatar: string;
 };
+export type Doc = {
+  location: string;
+  name: string;
+  language: string;
+  version: string;
+  description: string;
+  fullName: string;
+  date: string;
+  docLink: string;
+  model: string;
+};
+
+export type PromptProps = {
+  prompts: { name: string; id: string; type: string }[];
+  selectedPrompt: { name: string; id: string; type: string };
+  onSelectPrompt: (name: string, id: string, type: string) => void;
+  setPrompts: (prompts: { name: string; id: string; type: string }[]) => void;
+  apiHost: string;
+};
+
+export type DocumentsProps = {
+  documents: Doc[] | null;
+  handleDeleteDocument: (index: number, document: Doc) => void;
+};
+
+export type CreateAPIKeyModalProps = {
+  close: () => void;
+  createAPIKey: (payload: {
+    name: string;
+    source: string;
+    prompt_id: string;
+    chunks: string;
+  }) => void;
+};
+
+export type SaveAPIKeyModalProps = {
+  apiKey: string;
+  close: () => void;
+};

frontend/src/preferences/PromptsModal.tsx

@@ -0,0 +1,222 @@
+import { ActiveState } from '../models/misc';
+import Exit from '../assets/exit.svg';
+
+function AddPrompt({
+  setModalState,
+  handleAddPrompt,
+  newPromptName,
+  setNewPromptName,
+  newPromptContent,
+  setNewPromptContent,
+}: {
+  setModalState: (state: ActiveState) => void;
+  handleAddPrompt?: () => void;
+  newPromptName: string;
+  setNewPromptName: (name: string) => void;
+  newPromptContent: string;
+  setNewPromptContent: (content: string) => void;
+}) {
+  return (
+    <div className="relative">
+      <button
+        className="absolute top-3 right-4 m-2 w-3"
+        onClick={() => {
+          setModalState('INACTIVE');
+        }}
+      >
+        <img className="filter dark:invert" src={Exit} />
+      </button>
+      <div className="p-8">
+        <p className="mb-1 text-xl text-jet dark:text-bright-gray">
+          Add Prompt
+        </p>
+        <p className="mb-7 text-xs text-[#747474] dark:text-[#7F7F82]">
+          Add your custom prompt and save it to DocsGPT
+        </p>
+        <div>
+          <input
+            placeholder="Prompt Name"
+            type="text"
+            className="h-10 w-full rounded-lg border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+            value={newPromptName}
+            onChange={(e) => setNewPromptName(e.target.value)}
+          ></input>
+          <div className="relative bottom-12 left-3 mt-[-3.00px]">
+            <span className="bg-white px-1 text-xs text-silver dark:bg-outer-space dark:text-silver">
+              Prompt Name
+            </span>
+          </div>
+          <div className="relative top-[7px] left-3">
+            <span className="bg-white px-1 text-xs text-silver dark:bg-outer-space dark:text-silver">
+              Prompt Text
+            </span>
+          </div>
+          <textarea
+            className="h-56 w-full rounded-lg border-2 border-silver px-3 py-2 outline-none dark:bg-transparent dark:text-silver"
+            value={newPromptContent}
+            onChange={(e) => setNewPromptContent(e.target.value)}
+          ></textarea>
+        </div>
+        <div className="mt-6 flex flex-row-reverse">
+          <button
+            onClick={handleAddPrompt}
+            className="rounded-3xl bg-purple-30 px-5 py-2 text-sm text-white transition-all hover:opacity-90"
+          >
+            Save
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+}
+
+function EditPrompt({
+  setModalState,
+  handleEditPrompt,
+  editPromptName,
+  setEditPromptName,
+  editPromptContent,
+  setEditPromptContent,
+  currentPromptEdit,
+}: {
+  setModalState: (state: ActiveState) => void;
+  handleEditPrompt?: (id: string, type: string) => void;
+  editPromptName: string;
+  setEditPromptName: (name: string) => void;
+  editPromptContent: string;
+  setEditPromptContent: (content: string) => void;
+  currentPromptEdit: { name: string; id: string; type: string };
+}) {
+  return (
+    <div className="relative">
+      <button
+        className="absolute top-3 right-4 m-2 w-3"
+        onClick={() => {
+          setModalState('INACTIVE');
+        }}
+      >
+        <img className="filter dark:invert" src={Exit} />
+      </button>
+      <div className="p-8">
+        <p className="mb-1 text-xl text-jet dark:text-bright-gray">
+          Edit Prompt
+        </p>
+        <p className="mb-7 text-xs text-[#747474] dark:text-[#7F7F82]">
+          Edit your custom prompt and save it to DocsGPT
+        </p>
+        <div>
+          <input
+            placeholder="Prompt Name"
+            type="text"
+            className="h-10 w-full rounded-lg border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+            value={editPromptName}
+            onChange={(e) => setEditPromptName(e.target.value)}
+          ></input>
+          <div className="relative bottom-12 left-3 mt-[-3.00px]">
+            <span className="bg-white px-1 text-xs text-silver dark:bg-outer-space dark:text-silver">
+              Prompt Name
+            </span>
+          </div>
+          <div className="relative top-[7px] left-3">
+            <span className="bg-white px-1 text-xs text-silver dark:bg-outer-space dark:text-silver">
+              Prompt Text
+            </span>
+          </div>
+          <textarea
+            className="h-56 w-full rounded-lg border-2 border-silver px-3 py-2 outline-none dark:bg-transparent dark:text-silver"
+            value={editPromptContent}
+            onChange={(e) => setEditPromptContent(e.target.value)}
+          ></textarea>
+        </div>
+        <div className="mt-6 flex flex-row-reverse gap-4">
+          <button
+            className={`rounded-3xl bg-purple-30 px-5 py-2 text-sm text-white transition-all ${
+              currentPromptEdit.type === 'public'
+                ? 'cursor-not-allowed opacity-50'
+                : 'hover:opacity-90'
+            }`}
+            onClick={() => {
+              handleEditPrompt &&
+                handleEditPrompt(currentPromptEdit.id, currentPromptEdit.type);
+            }}
+            disabled={currentPromptEdit.type === 'public'}
+          >
+            Save
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+}
+
+export default function PromptsModal({
+  modalState,
+  setModalState,
+  type,
+  newPromptName,
+  setNewPromptName,
+  newPromptContent,
+  setNewPromptContent,
+  editPromptName,
+  setEditPromptName,
+  editPromptContent,
+  setEditPromptContent,
+  currentPromptEdit,
+  handleAddPrompt,
+  handleEditPrompt,
+}: {
+  modalState: ActiveState;
+  setModalState: (state: ActiveState) => void;
+  type: 'ADD' | 'EDIT';
+  newPromptName: string;
+  setNewPromptName: (name: string) => void;
+  newPromptContent: string;
+  setNewPromptContent: (content: string) => void;
+  editPromptName: string;
+  setEditPromptName: (name: string) => void;
+  editPromptContent: string;
+  setEditPromptContent: (content: string) => void;
+  currentPromptEdit: { name: string; id: string; type: string };
+  handleAddPrompt?: () => void;
+  handleEditPrompt?: (id: string, type: string) => void;
+}) {
+  let view;
+
+  if (type === 'ADD') {
+    view = (
+      <AddPrompt
+        setModalState={setModalState}
+        handleAddPrompt={handleAddPrompt}
+        newPromptName={newPromptName}
+        setNewPromptName={setNewPromptName}
+        newPromptContent={newPromptContent}
+        setNewPromptContent={setNewPromptContent}
+      />
+    );
+  } else if (type === 'EDIT') {
+    view = (
+      <EditPrompt
+        setModalState={setModalState}
+        handleEditPrompt={handleEditPrompt}
+        editPromptName={editPromptName}
+        setEditPromptName={setEditPromptName}
+        editPromptContent={editPromptContent}
+        setEditPromptContent={setEditPromptContent}
+        currentPromptEdit={currentPromptEdit}
+      />
+    );
+  } else {
+    view = <></>;
+  }
+  return (
+    <article
+      className={`${
+        modalState === 'ACTIVE' ? 'visible' : 'hidden'
+      } fixed top-0 left-0 z-30  h-screen w-screen  bg-gray-alpha`}
+    >
+      <article className="mx-auto mt-24 flex w-[90vw] max-w-lg  flex-col gap-4 rounded-2xl bg-white shadow-lg dark:bg-outer-space">
+        {view}
+      </article>
+    </article>
+  );
+}

frontend/src/preferences/preferenceSlice.ts

@@ -10,6 +10,7 @@ interface Preference {
   apiKey: string;
   prompt: { name: string; id: string; type: string };
   selectedDocs: Doc | null;
+  chunks: string;
   sourceDocs: Doc[] | null;
   conversations: { name: string; id: string }[] | null;
 }
@@ -17,6 +18,7 @@ interface Preference {
 const initialState: Preference = {
   apiKey: 'xxx',
   prompt: { name: 'default', id: 'default', type: 'public' },
+  chunks: '2',
   selectedDocs: {
     name: 'default',
     language: 'default',
@@ -51,6 +53,9 @@ export const prefSlice = createSlice({
     setPrompt: (state, action) => {
       state.prompt = action.payload;
     },
+    setChunks: (state, action) => {
+      state.chunks = action.payload;
+    },
   },
 });
 
@@ -60,6 +65,7 @@ export const {
   setSourceDocs,
   setConversations,
   setPrompt,
+  setChunks,
 } = prefSlice.actions;
 export default prefSlice.reducer;
 
@@ -91,6 +97,16 @@ prefListenerMiddleware.startListening({
   },
 });
 
+prefListenerMiddleware.startListening({
+  matcher: isAnyOf(setChunks),
+  effect: (action, listenerApi) => {
+    localStorage.setItem(
+      'DocsGPTChunks',
+      JSON.stringify((listenerApi.getState() as RootState).preference.chunks),
+    );
+  },
+});
+
 export const selectApiKey = (state: RootState) => state.preference.apiKey;
 export const selectApiKeyStatus = (state: RootState) =>
   !!state.preference.apiKey;
@@ -105,3 +121,4 @@ export const selectConversations = (state: RootState) =>
 export const selectConversationId = (state: RootState) =>
   state.conversation.conversationId;
 export const selectPrompt = (state: RootState) => state.preference.prompt;
+export const selectChunks = (state: RootState) => state.preference.chunks;

frontend/src/settings/APIKeys.tsx

@@ -0,0 +1,336 @@
+import React from 'react';
+import { useSelector } from 'react-redux';
+import Dropdown from '../components/Dropdown';
+import {
+  Doc,
+  CreateAPIKeyModalProps,
+  SaveAPIKeyModalProps,
+} from '../models/misc';
+import { selectSourceDocs } from '../preferences/preferenceSlice';
+import Exit from '../assets/exit.svg';
+import Trash from '../assets/trash.svg';
+
+const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+const embeddingsName =
+  import.meta.env.VITE_EMBEDDINGS_NAME ||
+  'huggingface_sentence-transformers/all-mpnet-base-v2';
+
+const APIKeys: React.FC = () => {
+  const [isCreateModalOpen, setCreateModal] = React.useState(false);
+  const [isSaveKeyModalOpen, setSaveKeyModal] = React.useState(false);
+  const [newKey, setNewKey] = React.useState('');
+  const [apiKeys, setApiKeys] = React.useState<
+    { name: string; key: string; source: string; id: string }[]
+  >([]);
+  const handleDeleteKey = (id: string) => {
+    fetch(`${apiHost}/api/delete_api_key`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ id }),
+    })
+      .then((response) => {
+        if (!response.ok) {
+          throw new Error('Failed to delete API Key');
+        }
+        return response.json();
+      })
+      .then((data) => {
+        data.status === 'ok' &&
+          setApiKeys((previous) => previous.filter((elem) => elem.id !== id));
+      })
+      .catch((error) => {
+        console.error(error);
+      });
+  };
+  React.useEffect(() => {
+    fetchAPIKeys();
+  }, []);
+  const fetchAPIKeys = async () => {
+    try {
+      const response = await fetch(`${apiHost}/api/get_api_keys`);
+      if (!response.ok) {
+        throw new Error('Failed to fetch API Keys');
+      }
+      const apiKeys = await response.json();
+      setApiKeys(apiKeys);
+    } catch (error) {
+      console.log(error);
+    }
+  };
+  const createAPIKey = (payload: {
+    name: string;
+    source: string;
+    prompt_id: string;
+    chunks: string;
+  }) => {
+    fetch(`${apiHost}/api/create_api_key`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(payload),
+    })
+      .then((response) => {
+        if (!response.ok) {
+          throw new Error('Failed to create API Key');
+        }
+        return response.json();
+      })
+      .then((data) => {
+        setApiKeys([...apiKeys, data]);
+        setCreateModal(false);
+        setNewKey(data.key);
+        setSaveKeyModal(true);
+        fetchAPIKeys();
+      })
+      .catch((error) => {
+        console.error(error);
+      });
+  };
+  return (
+    <div className="mt-8">
+      <div className="flex w-full flex-col lg:w-max">
+        <div className="flex justify-end">
+          <button
+            onClick={() => setCreateModal(true)}
+            className="rounded-full bg-purple-30 px-4 py-3 text-sm text-white hover:opacity-90"
+          >
+            Create new
+          </button>
+        </div>
+        {isCreateModalOpen && (
+          <CreateAPIKeyModal
+            close={() => setCreateModal(false)}
+            createAPIKey={createAPIKey}
+          />
+        )}
+        {isSaveKeyModalOpen && (
+          <SaveAPIKeyModal
+            apiKey={newKey}
+            close={() => setSaveKeyModal(false)}
+          />
+        )}
+        <div className="mt-[27px] w-full">
+          <div className="w-full overflow-x-auto">
+            <table className="block w-max table-auto content-center justify-center rounded-xl border text-center dark:border-chinese-silver dark:text-bright-gray">
+              <thead>
+                <tr>
+                  <th className="border-r p-4 md:w-[244px]">Name</th>
+                  <th className="w-[244px] border-r px-4 py-2">
+                    Source document
+                  </th>
+                  <th className="w-[244px] border-r px-4 py-2">API Key</th>
+                  <th className="px-4 py-2"></th>
+                </tr>
+              </thead>
+              <tbody>
+                {apiKeys?.map((element, index) => (
+                  <tr key={index}>
+                    <td className="border-r border-t p-4">{element.name}</td>
+                    <td className="border-r border-t p-4">{element.source}</td>
+                    <td className="border-r border-t p-4">{element.key}</td>
+                    <td className="border-t p-4">
+                      <img
+                        src={Trash}
+                        alt="Delete"
+                        className="h-4 w-4 cursor-pointer hover:opacity-50"
+                        id={`img-${index}`}
+                        onClick={() => handleDeleteKey(element.id)}
+                      />
+                    </td>
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+const CreateAPIKeyModal: React.FC<CreateAPIKeyModalProps> = ({
+  close,
+  createAPIKey,
+}) => {
+  const [APIKeyName, setAPIKeyName] = React.useState<string>('');
+  const [sourcePath, setSourcePath] = React.useState<{
+    label: string;
+    value: string;
+  } | null>(null);
+
+  const chunkOptions = ['0', '2', '4', '6', '8', '10'];
+  const [chunk, setChunk] = React.useState<string>('2');
+  const [activePrompts, setActivePrompts] = React.useState<
+    { name: string; id: string; type: string }[]
+  >([]);
+  const [prompt, setPrompt] = React.useState<{
+    name: string;
+    id: string;
+    type: string;
+  } | null>(null);
+  const docs = useSelector(selectSourceDocs);
+  React.useEffect(() => {
+    const fetchPrompts = async () => {
+      try {
+        const response = await fetch(`${apiHost}/api/get_prompts`);
+        if (!response.ok) {
+          throw new Error('Failed to fetch prompts');
+        }
+        const promptsData = await response.json();
+        setActivePrompts(promptsData);
+      } catch (error) {
+        console.error(error);
+      }
+    };
+    fetchPrompts();
+  }, []);
+  const extractDocPaths = () =>
+    docs
+      ? docs
+          .filter((doc) => doc.model === embeddingsName)
+          .map((doc: Doc) => {
+            let namePath = doc.name;
+            if (doc.language === namePath) {
+              namePath = '.project';
+            }
+            let docPath = 'default';
+            if (doc.location === 'local') {
+              docPath = 'local' + '/' + doc.name + '/';
+            } else if (doc.location === 'remote') {
+              docPath =
+                doc.language +
+                '/' +
+                namePath +
+                '/' +
+                doc.version +
+                '/' +
+                doc.model +
+                '/';
+            }
+            return {
+              label: doc.name,
+              value: docPath,
+            };
+          })
+      : [];
+
+  return (
+    <div className="fixed top-0 left-0 z-30 flex h-screen w-screen items-center justify-center bg-gray-alpha bg-opacity-50">
+      <div className="relative w-11/12 rounded-2xl bg-white p-10 dark:bg-outer-space sm:w-[512px]">
+        <button className="absolute top-3 right-4 m-2 w-3" onClick={close}>
+          <img className="filter dark:invert" src={Exit} />
+        </button>
+        <div className="mb-6">
+          <span className="text-xl text-jet dark:text-bright-gray">
+            Create New API Key
+          </span>
+        </div>
+        <div className="relative mt-5 mb-4">
+          <span className="absolute left-2 -top-2 bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+            API Key Name
+          </span>
+          <input
+            type="text"
+            className="h-10 w-full rounded-md border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+            value={APIKeyName}
+            onChange={(e) => setAPIKeyName(e.target.value)}
+          />
+        </div>
+        <div className="my-4">
+          <Dropdown
+            placeholder="Source document"
+            selectedValue={sourcePath}
+            onSelect={(selection: { label: string; value: string }) =>
+              setSourcePath(selection)
+            }
+            options={extractDocPaths()}
+            size="w-full"
+            rounded="xl"
+          />
+        </div>
+        <div className="my-4">
+          <Dropdown
+            options={activePrompts}
+            selectedValue={prompt ? prompt.name : null}
+            placeholder="Select active prompt"
+            onSelect={(value: { name: string; id: string; type: string }) =>
+              setPrompt(value)
+            }
+            size="w-full"
+          />
+        </div>
+        <div className="my-4">
+          <p className="mb-2 ml-2 font-bold text-jet dark:text-bright-gray">
+            Chunks processed per query
+          </p>
+          <Dropdown
+            options={chunkOptions}
+            selectedValue={chunk}
+            onSelect={(value: string) => setChunk(value)}
+            size="w-full"
+          />
+        </div>
+        <button
+          disabled={!sourcePath || APIKeyName.length === 0 || !prompt}
+          onClick={() =>
+            sourcePath &&
+            prompt &&
+            createAPIKey({
+              name: APIKeyName,
+              source: sourcePath.value,
+              prompt_id: prompt.id,
+              chunks: chunk,
+            })
+          }
+          className="float-right mt-4 rounded-full bg-purple-30 px-4 py-3 text-white disabled:opacity-50"
+        >
+          Create
+        </button>
+      </div>
+    </div>
+  );
+};
+
+const SaveAPIKeyModal: React.FC<SaveAPIKeyModalProps> = ({ apiKey, close }) => {
+  const [isCopied, setIsCopied] = React.useState(false);
+  const handleCopyKey = () => {
+    navigator.clipboard.writeText(apiKey);
+    setIsCopied(true);
+  };
+  return (
+    <div className="fixed top-0 left-0 z-30 flex h-screen w-screen items-center justify-center bg-gray-alpha bg-opacity-50">
+      <div className="relative w-11/12 rounded-3xl bg-white px-6 py-8 dark:bg-outer-space dark:text-bright-gray sm:w-[512px]">
+        <button className="absolute top-3 right-4 m-2 w-3" onClick={close}>
+          <img className="filter dark:invert" src={Exit} />
+        </button>
+        <h1 className="my-0 text-xl font-medium">Please save your Key</h1>
+        <h3 className="text-sm font-normal text-outer-space">
+          This is the only time your key will be shown.
+        </h3>
+        <div className="flex justify-between py-2">
+          <div>
+            <h2 className="text-base font-semibold">API Key</h2>
+            <span className="text-sm font-normal leading-7 ">{apiKey}</span>
+          </div>
+          <button
+            className="my-1 h-10 w-20 rounded-full border border-purple-30 p-2 text-sm text-purple-30 hover:bg-purple-30 hover:text-white dark:border-purple-500 dark:text-purple-500"
+            onClick={handleCopyKey}
+          >
+            {isCopied ? 'Copied' : 'Copy'}
+          </button>
+        </div>
+        <button
+          onClick={close}
+          className="rounded-full bg-philippine-yellow px-4 py-3 font-medium text-black hover:bg-[#E6B91A]"
+        >
+          I saved the Key
+        </button>
+      </div>
+    </div>
+  );
+};
+
+export default APIKeys;

frontend/src/settings/Documents.tsx

@@ -0,0 +1,60 @@
+import { DocumentsProps } from '../models/misc';
+import Trash from '../assets/trash.svg';
+
+const Documents: React.FC<DocumentsProps> = ({
+  documents,
+  handleDeleteDocument,
+}) => {
+  return (
+    <div className="mt-8">
+      <div className="flex flex-col">
+        <div className="mt-[27px] w-max overflow-x-auto rounded-xl border dark:border-chinese-silver">
+          <table className="block w-full table-auto content-center justify-center text-center dark:text-bright-gray">
+            <thead>
+              <tr>
+                <th className="border-r p-4 md:w-[244px]">Document Name</th>
+                <th className="w-[244px] border-r px-4 py-2">Vector Date</th>
+                <th className="w-[244px] border-r px-4 py-2">Type</th>
+                <th className="px-4 py-2"></th>
+              </tr>
+            </thead>
+            <tbody>
+              {documents &&
+                documents.map((document, index) => (
+                  <tr key={index}>
+                    <td className="border-r border-t px-4 py-2">
+                      {document.name}
+                    </td>
+                    <td className="border-r border-t px-4 py-2">
+                      {document.date}
+                    </td>
+                    <td className="border-r border-t px-4 py-2">
+                      {document.location === 'remote'
+                        ? 'Pre-loaded'
+                        : 'Private'}
+                    </td>
+                    <td className="border-t px-4 py-2">
+                      {document.location !== 'remote' && (
+                        <img
+                          src={Trash}
+                          alt="Delete"
+                          className="h-4 w-4 cursor-pointer hover:opacity-50"
+                          id={`img-${index}`}
+                          onClick={(event) => {
+                            event.stopPropagation();
+                            handleDeleteDocument(index, document);
+                          }}
+                        />
+                      )}
+                    </td>
+                  </tr>
+                ))}
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default Documents;

frontend/src/settings/General.tsx

@@ -0,0 +1,100 @@
+import React from 'react';
+import { useSelector, useDispatch } from 'react-redux';
+import Prompts from './Prompts';
+import { useDarkTheme } from '../hooks';
+import Dropdown from '../components/Dropdown';
+import {
+  selectPrompt,
+  setPrompt,
+  setChunks,
+  selectChunks,
+} from '../preferences/preferenceSlice';
+
+const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+
+const General: React.FC = () => {
+  const themes = ['Light', 'Dark'];
+  const languages = ['English'];
+  const chunks = ['0', '2', '4', '6', '8', '10'];
+  const [prompts, setPrompts] = React.useState<
+    { name: string; id: string; type: string }[]
+  >([]);
+  const selectedChunks = useSelector(selectChunks);
+  const [isDarkTheme, toggleTheme] = useDarkTheme();
+  const [selectedTheme, setSelectedTheme] = React.useState(
+    isDarkTheme ? 'Dark' : 'Light',
+  );
+  const dispatch = useDispatch();
+  const [selectedLanguage, setSelectedLanguage] = React.useState(languages[0]);
+  const selectedPrompt = useSelector(selectPrompt);
+
+  React.useEffect(() => {
+    const fetchPrompts = async () => {
+      try {
+        const response = await fetch(`${apiHost}/api/get_prompts`);
+        if (!response.ok) {
+          throw new Error('Failed to fetch prompts');
+        }
+        const promptsData = await response.json();
+        setPrompts(promptsData);
+      } catch (error) {
+        console.error(error);
+      }
+    };
+    fetchPrompts();
+  }, []);
+  return (
+    <div className="mt-[59px]">
+      <div className="mb-4">
+        <p className="font-bold text-jet dark:text-bright-gray">Select Theme</p>
+        <Dropdown
+          options={themes}
+          selectedValue={selectedTheme}
+          onSelect={(option: string) => {
+            setSelectedTheme(option);
+            option !== selectedTheme && toggleTheme();
+          }}
+          size="w-56"
+          rounded="3xl"
+        />
+      </div>
+      <div className="mb-4">
+        <p className="font-bold text-jet dark:text-bright-gray">
+          Select Language
+        </p>
+        <Dropdown
+          options={languages}
+          selectedValue={selectedLanguage}
+          onSelect={setSelectedLanguage}
+          size="w-56"
+          rounded="3xl"
+        />
+      </div>
+      <div className="mb-4">
+        <p className="font-bold text-jet dark:text-bright-gray">
+          Chunks processed per query
+        </p>
+        <Dropdown
+          options={chunks}
+          selectedValue={selectedChunks}
+          onSelect={(value: string) => dispatch(setChunks(value))}
+          size="w-56"
+          rounded="3xl"
+        />
+      </div>
+      <div>
+        <Prompts
+          prompts={prompts}
+          selectedPrompt={selectedPrompt}
+          onSelectPrompt={(name, id, type) =>
+            dispatch(setPrompt({ name: name, id: id, type: type }))
+          }
+          setPrompts={setPrompts}
+          apiHost={apiHost}
+        />
+      </div>
+    </div>
+  );
+};
+
+export default General;

frontend/src/settings/Prompts.tsx

@@ -0,0 +1,219 @@
+import React from 'react';
+import { PromptProps, ActiveState } from '../models/misc';
+import Dropdown from '../components/Dropdown';
+import PromptsModal from '../preferences/PromptsModal';
+
+const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+
+const Prompts: React.FC<PromptProps> = ({
+  prompts,
+  selectedPrompt,
+  onSelectPrompt,
+  setPrompts,
+}) => {
+  const handleSelectPrompt = ({
+    name,
+    id,
+    type,
+  }: {
+    name: string;
+    id: string;
+    type: string;
+  }) => {
+    setEditPromptName(name);
+    onSelectPrompt(name, id, type);
+  };
+  const [newPromptName, setNewPromptName] = React.useState('');
+  const [newPromptContent, setNewPromptContent] = React.useState('');
+  const [editPromptName, setEditPromptName] = React.useState('');
+  const [editPromptContent, setEditPromptContent] = React.useState('');
+  const [currentPromptEdit, setCurrentPromptEdit] = React.useState({
+    id: '',
+    name: '',
+    type: '',
+  });
+  const [modalType, setModalType] = React.useState<'ADD' | 'EDIT'>('ADD');
+  const [modalState, setModalState] = React.useState<ActiveState>('INACTIVE');
+
+  const handleAddPrompt = async () => {
+    try {
+      const response = await fetch(`${apiHost}/api/create_prompt`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          name: newPromptName,
+          content: newPromptContent,
+        }),
+      });
+      if (!response.ok) {
+        throw new Error('Failed to add prompt');
+      }
+      const newPrompt = await response.json();
+      if (setPrompts) {
+        setPrompts([
+          ...prompts,
+          { name: newPromptName, id: newPrompt.id, type: 'private' },
+        ]);
+      }
+      setModalState('INACTIVE');
+      onSelectPrompt(newPromptName, newPrompt.id, newPromptContent);
+      setNewPromptName(newPromptName);
+    } catch (error) {
+      console.error(error);
+    }
+  };
+
+  const handleDeletePrompt = (id: string) => {
+    setPrompts(prompts.filter((prompt) => prompt.id !== id));
+    fetch(`${apiHost}/api/delete_prompt`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ id: id }),
+    })
+      .then((response) => {
+        if (!response.ok) {
+          throw new Error('Failed to delete prompt');
+        }
+        // get 1st prompt and set it as selected
+        if (prompts.length > 0) {
+          onSelectPrompt(prompts[0].name, prompts[0].id, prompts[0].type);
+        }
+      })
+      .catch((error) => {
+        console.error(error);
+      });
+  };
+
+  const fetchPromptContent = async (id: string) => {
+    console.log('fetching prompt content');
+    try {
+      const response = await fetch(
+        `${apiHost}/api/get_single_prompt?id=${id}`,
+        {
+          method: 'GET',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+        },
+      );
+      if (!response.ok) {
+        throw new Error('Failed to fetch prompt content');
+      }
+      const promptContent = await response.json();
+      setEditPromptContent(promptContent.content);
+    } catch (error) {
+      console.error(error);
+    }
+  };
+
+  const handleSaveChanges = (id: string, type: string) => {
+    fetch(`${apiHost}/api/update_prompt`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        id: id,
+        name: editPromptName,
+        content: editPromptContent,
+      }),
+    })
+      .then((response) => {
+        if (!response.ok) {
+          throw new Error('Failed to update prompt');
+        }
+        if (setPrompts) {
+          const existingPromptIndex = prompts.findIndex(
+            (prompt) => prompt.id === id,
+          );
+          if (existingPromptIndex === -1) {
+            setPrompts([
+              ...prompts,
+              { name: editPromptName, id: id, type: type },
+            ]);
+          } else {
+            const updatedPrompts = [...prompts];
+            updatedPrompts[existingPromptIndex] = {
+              name: editPromptName,
+              id: id,
+              type: type,
+            };
+            setPrompts(updatedPrompts);
+          }
+        }
+        setModalState('INACTIVE');
+        onSelectPrompt(editPromptName, id, type);
+      })
+      .catch((error) => {
+        console.error(error);
+      });
+  };
+
+  return (
+    <>
+      <div>
+        <div className="mb-4 flex flex-row items-center gap-8">
+          <div>
+            <p className="font-semibold dark:text-bright-gray">Active Prompt</p>
+            <Dropdown
+              options={prompts}
+              selectedValue={selectedPrompt.name}
+              onSelect={handleSelectPrompt}
+              size="w-56"
+              rounded="3xl"
+              showEdit
+              showDelete
+              onEdit={({
+                id,
+                name,
+                type,
+              }: {
+                id: string;
+                name: string;
+                type: string;
+              }) => {
+                setModalType('EDIT');
+                setEditPromptName(name);
+                fetchPromptContent(id);
+                setCurrentPromptEdit({ id: id, name: name, type: type });
+                setModalState('ACTIVE');
+              }}
+              onDelete={handleDeletePrompt}
+            />
+          </div>
+          <button
+            className="mt-[24px] rounded-3xl border-2 border-solid border-purple-30 px-5 py-3 text-purple-30 hover:bg-purple-30 hover:text-white"
+            onClick={() => {
+              setModalType('ADD');
+              setModalState('ACTIVE');
+            }}
+          >
+            Add new
+          </button>
+        </div>
+      </div>
+      <PromptsModal
+        type={modalType}
+        modalState={modalState}
+        setModalState={setModalState}
+        newPromptName={newPromptName}
+        setNewPromptName={setNewPromptName}
+        newPromptContent={newPromptContent}
+        setNewPromptContent={setNewPromptContent}
+        editPromptName={editPromptName}
+        setEditPromptName={setEditPromptName}
+        editPromptContent={editPromptContent}
+        setEditPromptContent={setEditPromptContent}
+        currentPromptEdit={currentPromptEdit}
+        handleAddPrompt={handleAddPrompt}
+        handleEditPrompt={handleSaveChanges}
+      />
+    </>
+  );
+};
+
+export default Prompts;

frontend/src/settings/Widgets.tsx

@@ -0,0 +1,112 @@
+import React from 'react';
+import Dropdown from '../components/Dropdown';
+
+const Widgets: React.FC<{
+  widgetScreenshot: File | null;
+  onWidgetScreenshotChange: (screenshot: File | null) => void;
+}> = ({ widgetScreenshot, onWidgetScreenshotChange }) => {
+  const widgetSources = ['Source 1', 'Source 2', 'Source 3'];
+  const widgetMethods = ['Method 1', 'Method 2', 'Method 3'];
+  const widgetTypes = ['Type 1', 'Type 2', 'Type 3'];
+
+  const [selectedWidgetSource, setSelectedWidgetSource] = React.useState(
+    widgetSources[0],
+  );
+  const [selectedWidgetMethod, setSelectedWidgetMethod] = React.useState(
+    widgetMethods[0],
+  );
+  const [selectedWidgetType, setSelectedWidgetType] = React.useState(
+    widgetTypes[0],
+  );
+
+  // const [widgetScreenshot, setWidgetScreenshot] = useState<File | null>(null);
+  const [widgetCode, setWidgetCode] = React.useState<string>(''); // Your widget code state
+
+  const handleScreenshotChange = (
+    event: React.ChangeEvent<HTMLInputElement>,
+  ) => {
+    const files = event.target.files;
+
+    if (files && files.length > 0) {
+      const selectedScreenshot = files[0];
+      onWidgetScreenshotChange(selectedScreenshot); // Update the screenshot in the parent component
+    }
+  };
+
+  const handleCopyToClipboard = () => {
+    // Create a new textarea element to select the text
+    const textArea = document.createElement('textarea');
+    textArea.value = widgetCode;
+    document.body.appendChild(textArea);
+
+    // Select and copy the text
+    textArea.select();
+    document.execCommand('copy');
+
+    // Clean up the textarea element
+    document.body.removeChild(textArea);
+  };
+
+  return (
+    <div>
+      <div className="mt-[59px]">
+        <p className="font-bold text-jet">Widget Source</p>
+        <Dropdown
+          options={widgetSources}
+          selectedValue={selectedWidgetSource}
+          onSelect={setSelectedWidgetSource}
+        />
+      </div>
+      <div className="mt-5">
+        <p className="font-bold text-jet">Widget Method</p>
+        <Dropdown
+          options={widgetMethods}
+          selectedValue={selectedWidgetMethod}
+          onSelect={setSelectedWidgetMethod}
+        />
+      </div>
+      <div className="mt-5">
+        <p className="font-bold text-jet">Widget Type</p>
+        <Dropdown
+          options={widgetTypes}
+          selectedValue={selectedWidgetType}
+          onSelect={setSelectedWidgetType}
+        />
+      </div>
+      <div className="mt-6">
+        <p className="font-bold text-jet">Widget Code Snippet</p>
+        <textarea
+          rows={4}
+          value={widgetCode}
+          onChange={(e) => setWidgetCode(e.target.value)}
+          className="mt-3 w-full rounded-lg border-2 p-2"
+        />
+      </div>
+      <div className="mt-1">
+        <button
+          onClick={handleCopyToClipboard}
+          className="rounded-lg bg-blue-400 px-2 py-2 font-bold text-white transition-all hover:bg-blue-600"
+        >
+          Copy
+        </button>
+      </div>
+
+      <div className="mt-4">
+        <p className="text-lg font-semibold">Widget Screenshot</p>
+        <input type="file" accept="image/*" onChange={handleScreenshotChange} />
+      </div>
+
+      {widgetScreenshot && (
+        <div className="mt-4">
+          <img
+            src={URL.createObjectURL(widgetScreenshot)}
+            alt="Widget Screenshot"
+            className="max-w-full rounded-lg border border-gray-300"
+          />
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default Widgets;

frontend/src/settings/index.tsx

@@ -0,0 +1,127 @@
+import React from 'react';
+import { useSelector, useDispatch } from 'react-redux';
+import General from './General';
+import Documents from './Documents';
+import APIKeys from './APIKeys';
+import Widgets from './Widgets';
+import {
+  selectSourceDocs,
+  setSourceDocs,
+} from '../preferences/preferenceSlice';
+import { Doc } from '../preferences/preferenceApi';
+import ArrowLeft from '../assets/arrow-left.svg';
+import ArrowRight from '../assets/arrow-right.svg';
+
+const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+
+const Settings: React.FC = () => {
+  const dispatch = useDispatch();
+  const tabs = ['General', 'Documents', 'API Keys'];
+  const [activeTab, setActiveTab] = React.useState('General');
+  const [widgetScreenshot, setWidgetScreenshot] = React.useState<File | null>(
+    null,
+  );
+
+  const documents = useSelector(selectSourceDocs);
+  const updateWidgetScreenshot = (screenshot: File | null) => {
+    setWidgetScreenshot(screenshot);
+  };
+  const handleDeleteClick = (index: number, doc: Doc) => {
+    const docPath = 'indexes/' + 'local' + '/' + doc.name;
+    fetch(`${apiHost}/api/delete_old?path=${docPath}`, {
+      method: 'GET',
+    })
+      .then((response) => {
+        if (response.ok && documents) {
+          const updatedDocuments = [
+            ...documents.slice(0, index),
+            ...documents.slice(index + 1),
+          ];
+          dispatch(setSourceDocs(updatedDocuments));
+        }
+      })
+      .catch((error) => console.error(error));
+  };
+  return (
+    <div className="wa p-4 pt-20 md:p-12">
+      <p className="text-2xl font-bold text-eerie-black dark:text-bright-gray">
+        Settings
+      </p>
+      <div className="mt-6 flex flex-row items-center space-x-4 overflow-x-auto md:space-x-8 ">
+        <div className="md:hidden">
+          <button
+            onClick={() => scrollTabs(-1)}
+            className="flex h-8 w-8 items-center justify-center rounded-full border-2 border-purple-30 transition-all hover:bg-gray-100"
+          >
+            <img src={ArrowLeft} alt="left-arrow" className="h-6 w-6" />
+          </button>
+        </div>
+        <div className="flex flex-nowrap space-x-4 overflow-x-auto md:space-x-8">
+          {tabs.map((tab, index) => (
+            <button
+              key={index}
+              onClick={() => setActiveTab(tab)}
+              className={`h-9 rounded-3xl px-4 font-bold ${
+                activeTab === tab
+                  ? 'bg-purple-3000 text-purple-30 dark:bg-dark-charcoal'
+                  : 'text-gray-6000'
+              }`}
+            >
+              {tab}
+            </button>
+          ))}
+        </div>
+        <div className="md:hidden">
+          <button
+            onClick={() => scrollTabs(1)}
+            className="flex h-8 w-8 items-center justify-center rounded-full border-2 border-purple-30 hover:bg-gray-100"
+          >
+            <img src={ArrowRight} alt="right-arrow" className="h-6 w-6" />
+          </button>
+        </div>
+      </div>
+      {renderActiveTab()}
+
+      {/* {activeTab === 'Widgets' && (
+        <Widgets
+          widgetScreenshot={widgetScreenshot}
+          onWidgetScreenshotChange={updateWidgetScreenshot}
+        />
+      )} */}
+    </div>
+  );
+
+  function scrollTabs(direction: number) {
+    const container = document.querySelector('.flex-nowrap');
+    if (container) {
+      container.scrollLeft += direction * 100; // Adjust the scroll amount as needed
+    }
+  }
+
+  function renderActiveTab() {
+    switch (activeTab) {
+      case 'General':
+        return <General />;
+      case 'Documents':
+        return (
+          <Documents
+            documents={documents}
+            handleDeleteDocument={handleDeleteClick}
+          />
+        );
+      case 'Widgets':
+        return (
+          <Widgets
+            widgetScreenshot={widgetScreenshot} // Add this line
+            onWidgetScreenshotChange={updateWidgetScreenshot} // Add this line
+          />
+        );
+      case 'API Keys':
+        return <APIKeys />;
+      default:
+        return null;
+    }
+  }
+};
+
+export default Settings;

frontend/src/store.ts

@@ -8,11 +8,13 @@ import {
 const key = localStorage.getItem('DocsGPTApiKey');
 const prompt = localStorage.getItem('DocsGPTPrompt');
 const doc = localStorage.getItem('DocsGPTRecentDocs');
+const chunks = localStorage.getItem('DocsGPTChunks');
 
 const store = configureStore({
   preloadedState: {
     preference: {
       apiKey: key ?? '',
+      chunks: JSON.parse(chunks ?? '2').toString(),
       selectedDocs: doc !== null ? JSON.parse(doc) : null,
       prompt:
         prompt !== null

frontend/src/upload/Upload.tsx

@@ -17,10 +17,18 @@ export default function Upload({
   const [docName, setDocName] = useState('');
   const [urlName, setUrlName] = useState('');
   const [url, setUrl] = useState('');
+  const [redditData, setRedditData] = useState({
+    client_id: '',
+    client_secret: '',
+    user_agent: '',
+    search_queries: [''],
+    number_posts: 10,
+  });
   const urlOptions: { label: string; value: string }[] = [
     { label: 'Crawler', value: 'crawler' },
     // { label: 'Sitemap', value: 'sitemap' },
     { label: 'Link', value: 'url' },
+    { label: 'Reddit', value: 'reddit' },
   ];
   const [urlType, setUrlType] = useState<{ label: string; value: string }>({
     label: 'Link',
@@ -163,7 +171,6 @@ export default function Upload({
   };
 
   const uploadRemote = () => {
-    console.log('here');
     const formData = new FormData();
     formData.append('name', urlName);
     formData.append('user', 'local');
@@ -171,6 +178,13 @@ export default function Upload({
       formData.append('source', urlType?.value);
     }
     formData.append('data', url);
+    if (
+      redditData.client_id.length > 0 &&
+      redditData.client_secret.length > 0
+    ) {
+      formData.set('name', 'other');
+      formData.set('data', JSON.stringify(redditData));
+    }
     const apiHost = import.meta.env.VITE_API_HOST;
     const xhr = new XMLHttpRequest();
     xhr.upload.addEventListener('progress', (event) => {
@@ -187,7 +201,7 @@ export default function Upload({
 
   const { getRootProps, getInputProps, isDragActive } = useDropzone({
     onDrop,
-    multiple: false,
+    multiple: true,
     onDragEnter: doNothing,
     onDragOver: doNothing,
     onDragLeave: doNothing,
@@ -202,6 +216,19 @@ export default function Upload({
         ['.docx'],
     },
   });
+  const handleChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const { name, value } = e.target;
+    if (name === 'search_queries' && value.length > 0) {
+      setRedditData({
+        ...redditData,
+        [name]: value.split(',').map((item) => item.trim()),
+      });
+    } else
+      setRedditData({
+        ...redditData,
+        [name]: value,
+      });
+  };
   let view;
   if (progress?.type === 'UPLOAD') {
     view = <UploadProgress></UploadProgress>;
@@ -280,31 +307,105 @@ export default function Upload({
               onSelect={(value: { label: string; value: string }) =>
                 setUrlType(value)
               }
+              size="w-full"
+              rounded="3xl"
             />
-            <input
-              placeholder="Enter name"
-              type="text"
-              className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
-              value={urlName}
-              onChange={(e) => setUrlName(e.target.value)}
-            ></input>
-            <div className="relative bottom-12 left-2 mt-[-18.39px]">
-              <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
-                Name
-              </span>
-            </div>
-            <input
-              placeholder="URL Link"
-              type="text"
-              className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
-              value={url}
-              onChange={(e) => setUrl(e.target.value)}
-            ></input>
-            <div className="relative bottom-12 left-2 mt-[-18.39px]">
-              <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
-                Link
-              </span>
-            </div>
+            {urlType.label !== 'Reddit' ? (
+              <>
+                <input
+                  placeholder="Enter name"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  value={urlName}
+                  onChange={(e) => setUrlName(e.target.value)}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Name
+                  </span>
+                </div>
+                <input
+                  placeholder="URL Link"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  value={url}
+                  onChange={(e) => setUrl(e.target.value)}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Link
+                  </span>
+                </div>
+              </>
+            ) : (
+              <>
+                <input
+                  placeholder="Enter client ID"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="client_id"
+                  value={redditData.client_id}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Client ID
+                  </span>
+                </div>
+                <input
+                  placeholder="Enter client secret"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="client_secret"
+                  value={redditData.client_secret}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Client secret
+                  </span>
+                </div>
+                <input
+                  placeholder="Enter user agent"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="user_agent"
+                  value={redditData.user_agent}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    User agent
+                  </span>
+                </div>
+                <input
+                  placeholder="Enter search queries"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="search_queries"
+                  value={redditData.search_queries}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Search queries
+                  </span>
+                </div>
+                <input
+                  placeholder="Enter number of posts"
+                  type="number"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="number_posts"
+                  value={redditData.number_posts}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Number of posts
+                  </span>
+                </div>
+              </>
+            )}
           </>
         )}
         <div className="flex flex-row-reverse">
@@ -318,7 +419,7 @@ export default function Upload({
             disabled={
               (files.length === 0 || docName.trim().length === 0) &&
               activeTab === 'file'
-            } // Disable the button if no file is selected or docName is empty
+            }
           >
             Train
           </button>
@@ -349,4 +450,3 @@ export default function Upload({
     </article>
   );
 }
-// TODO: sanitize all inputs

frontend/tailwind.config.cjs

@@ -46,7 +46,8 @@ module.exports = {
         'gun-metal':'#2E303E',
         'sonic-silver':'#747474',
         'soap':'#D8CCF1',
-        'independence':'#54546D'
+        'independence':'#54546D',
+        'philippine-yellow':'#FFC700',
       },
     },
   },

mock-backend/package-lock.json

@@ -357,9 +357,9 @@
       }
     },
     "node_modules/cookie": {
-      "version": "0.5.0",
-      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
-      "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
+      "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
       "engines": {
         "node": ">= 0.6"
       }
@@ -458,16 +458,16 @@
       }
     },
     "node_modules/express": {
-      "version": "4.18.2",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
-      "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
+      "version": "4.19.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz",
+      "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==",
       "dependencies": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
-        "body-parser": "1.20.1",
+        "body-parser": "1.20.2",
         "content-disposition": "0.5.4",
         "content-type": "~1.0.4",
-        "cookie": "0.5.0",
+        "cookie": "0.6.0",
         "cookie-signature": "1.0.6",
         "debug": "2.6.9",
         "depd": "2.0.0",
@@ -515,43 +515,6 @@
         "isarray": "0.0.1"
       }
     },
-    "node_modules/express/node_modules/body-parser": {
-      "version": "1.20.1",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
-      "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
-      "dependencies": {
-        "bytes": "3.1.2",
-        "content-type": "~1.0.4",
-        "debug": "2.6.9",
-        "depd": "2.0.0",
-        "destroy": "1.2.0",
-        "http-errors": "2.0.0",
-        "iconv-lite": "0.4.24",
-        "on-finished": "2.4.1",
-        "qs": "6.11.0",
-        "raw-body": "2.5.1",
-        "type-is": "~1.6.18",
-        "unpipe": "1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.8",
-        "npm": "1.2.8000 || >= 1.4.16"
-      }
-    },
-    "node_modules/express/node_modules/raw-body": {
-      "version": "2.5.1",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
-      "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
-      "dependencies": {
-        "bytes": "3.1.2",
-        "http-errors": "2.0.0",
-        "iconv-lite": "0.4.24",
-        "unpipe": "1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
     "node_modules/express/node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",

tests/llm/test_sagemaker.py

@@ -54,7 +54,7 @@ class TestSagemakerAPILLM(unittest.TestCase):
     def test_gen(self):
         with patch.object(self.sagemaker.runtime, 'invoke_endpoint', 
                           return_value=self.response) as mock_invoke_endpoint:
-            output = self.sagemaker.gen(None, None, self.messages)
+            output = self.sagemaker.gen(None, self.messages)
             mock_invoke_endpoint.assert_called_once_with(
                 EndpointName=self.sagemaker.endpoint,
                 ContentType='application/json',
@@ -66,7 +66,7 @@ class TestSagemakerAPILLM(unittest.TestCase):
     def test_gen_stream(self):
         with patch.object(self.sagemaker.runtime, 'invoke_endpoint_with_response_stream', 
                           return_value=self.response) as mock_invoke_endpoint:
-            output = list(self.sagemaker.gen_stream(None, None, self.messages))
+            output = list(self.sagemaker.gen_stream(None, self.messages))
             mock_invoke_endpoint.assert_called_once_with(
                 EndpointName=self.sagemaker.endpoint,
                 ContentType='application/json',