Merge pull request #1173 from sahil9001/system-theme-unless-set

feat: added theme check unless already set

.github/dependabot.yml

@@ -0,0 +1,15 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/application" # Location of package manifests
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "npm" # See documentation for possible values
+    directory: "/frontend" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/labeler.yml

@@ -4,6 +4,7 @@ on:
   - pull_request_target
 jobs:
   triage:
+    if: github.repository == 'arc53/DocsGPT'
     permissions:
       contents: read
       pull-requests: write

.github/workflows/pytest.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.11"]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -21,7 +21,7 @@ jobs:
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
       - name: Test with pytest and generate coverage report
         run: |
-          python -m pytest --cov=application --cov=scripts --cov=extensions --cov-report=xml
+          python -m pytest --cov=application --cov-report=xml
       - name: Upload coverage reports to Codecov
         if: github.event_name == 'pull_request' && matrix.python-version == '3.11'
         uses: codecov/codecov-action@v3

.gitignore

@@ -172,5 +172,5 @@ application/vectors/
 
 node_modules/
 .vscode/settings.json
-models/
+/models/
 model/

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Docker Debug Frontend",
+      "request": "launch",
+      "type": "chrome",
+      "preLaunchTask": "docker-compose: debug:frontend",
+      "url": "http://127.0.0.1:5173",
+      "webRoot": "${workspaceFolder}/frontend",
+      "skipFiles": [
+        "<node_internals>/**"
+      ]
+    }
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,21 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "docker-compose",
+      "label": "docker-compose: debug:frontend",
+      "dockerCompose": {
+        "up": {
+          "detached": true,
+          "services": [
+            "frontend"
+          ],
+          "build": true
+        },
+        "files": [
+          "${workspaceFolder}/docker-compose.yaml"
+        ]
+      }
+    }
+  ]
+}

HACKTOBERFEST.md

@@ -0,0 +1,37 @@
+# **🎉 Join the Hacktoberfest with DocsGPT and win a Free T-shirt and other prizes! 🎉**
+
+Welcome, contributors! We're excited to announce that DocsGPT is participating in Hacktoberfest. Get involved by submitting meaningful pull requests.
+
+All contributors with accepted PRs will receive a cool Holopin! 🤩 (Watch out for a reply in your PR to collect it).
+
+### 🏆 Top 50 contributors will recieve a special T-shirt
+
+### 🏆 [LLM Document analysis by LexEU competition](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md): 
+A separate competition is available for those sumbit best new retrieval / workflow method that will analyze a Document using EU laws.
+With 200$, 100$, 50$ prize for 1st, 2nd and 3rd place respectively.
+You can find more information [here](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md)
+
+## 📜 Here's How to Contribute:
+```text
+🛠️ Code: This is the golden ticket! Make meaningful contributions through PRs.
+
+🧩 API extention: Build an app utilising DocsGPT API. We prefer submissions that showcase original ideas and turn the API into an AI agent.
+
+Non-Code Contributions:
+
+📚 Wiki: Improve our documentation, Create a guide or change existing documentation.
+
+🖥️ Design: Improve the UI/UX or design a new feature.
+
+📝 Blogging or Content Creation: Write articles or create videos to showcase DocsGPT or highlight your contributions!
+```
+
+### 📝 Guidelines for Pull Requests:
+- Familiarize yourself with the current contributions and our [Roadmap](https://github.com/orgs/arc53/projects/2).
+- Before contributing we highly advise that you check existing [issues](https://github.com/arc53/DocsGPT/issues) or [create](https://github.com/arc53/DocsGPT/issues/new/choose) an issue and wait to get assigned.
+- Once you are finished with your contribution, please fill in this [form](https://airtable.com/appikMaJwdHhC1SDP/pagoblCJ9W29wf6Hf/form).
+- Refer to the [Documentation](https://docs.docsgpt.cloud/).
+- Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU) server. We're here to help newcomers, so don't hesitate to jump in! Join us [here](https://discord.gg/n5BX8dh8rU).
+  
+Thank you very much for considering contributing to DocsGPT during Hacktoberfest! 🙏 Your contributions (not just simple typo) could earn you a stylish new t-shirt and other prizes as a token of our appreciation. 🎁 Join us, and let's code together! 🚀
+

README.md

@@ -23,11 +23,15 @@ Say goodbye to time-consuming manual searches, and let <strong><a href="https://
  
 </div>
 
+### 🎃 [Hacktoberfest Prizes, Rules & Q&A](https://github.com/arc53/DocsGPT/blob/main/HACKTOBERFEST.md) 🎃
+
+### Our [Livestream to Dive into Hacktoberfest! Prizes, Rules & Q&A 🎉](https://www.youtube.com/watch?v=5QQaFFu9BC8) on 3rd of October 
+
 ### Production Support / Help for Companies:
 
 We're eager to provide personalized assistance when deploying your DocsGPT to a live environment.
 
-- [Get Enterprise / teams Demo :wave:](https://www.docsgpt.cloud/contact)
+- [Book Enterprise / teams Demo :wave:](https://cal.com/arc53/docsgpt-demo-b2b?date=2024-09-27&month=2024-09)
 - [Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
 
 ![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
@@ -46,9 +50,9 @@ You can find our roadmap [here](https://github.com/orgs/arc53/projects/2). Pleas
 
 If you don't have enough resources to run it, you can use bitsnbytes to quantize.
 
-## Features
+## End to End AI Framework for Information Retrieval
 
-![Main features of DocsGPT showcasing six main features](https://user-images.githubusercontent.com/17906039/220427472-2644cff4-7666-46a5-819f-fc4a521f63c7.png)
+![Architecture chart](https://github.com/user-attachments/assets/fc6a7841-ddfc-45e6-b5a0-d05fe648cbe2)
 
 ## Useful Links
 

application/Dockerfile

@@ -4,14 +4,11 @@ FROM ubuntu:24.04 as builder
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
-    apt-get install -y software-properties-common
-
-RUN add-apt-repository ppa:deadsnakes/ppa
-
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
 # Install necessary packages and Python
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends gcc curl wget unzip libc6-dev python3.11 python3.11-distutils python3.11-venv && \
-    apt-get clean && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends gcc wget unzip libc6-dev python3.11 python3.11-distutils python3.11-venv && \
     rm -rf /var/lib/apt/lists/* 
 
 # Verify Python installation and setup symlink
@@ -27,7 +24,7 @@ RUN wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.z
     rm mpnet-base-v2.zip
 
 # Install Rust
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y
 
 # Clean up to reduce container size
 RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
@@ -50,12 +47,10 @@ RUN pip install --no-cache-dir --upgrade pip && \
 FROM ubuntu:24.04 as final
 
 RUN apt-get update && \
-    apt-get install -y software-properties-common
-
-RUN add-apt-repository ppa:deadsnakes/ppa
-
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
 # Install Python
-RUN apt-get update && apt-get install -y --no-install-recommends python3.11 && \
+    apt-get update && apt-get install -y --no-install-recommends python3.11 && \
     ln -s /usr/bin/python3.11 /usr/bin/python && \
     rm -rf /var/lib/apt/lists/*
 

application/api/answer/routes.py

@@ -1,29 +1,38 @@
 import asyncio
+import datetime
+import json
+import logging
 import os
 import sys
-from flask import Blueprint, request, Response
-import json
-import datetime
-import logging
 import traceback
 
-from pymongo import MongoClient
+from bson.dbref import DBRef
 from bson.objectid import ObjectId
+from flask import Blueprint, current_app, make_response, request, Response
+from flask_restx import fields, Namespace, Resource
+
+from pymongo import MongoClient
 
 from application.core.settings import settings
+from application.error import bad_request
+from application.extensions import api
 from application.llm.llm_creator import LLMCreator
 from application.retriever.retriever_creator import RetrieverCreator
-from application.error import bad_request
+from application.utils import check_required_fields
 
 logger = logging.getLogger(__name__)
 
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 prompts_collection = db["prompts"]
 api_key_collection = db["api_keys"]
+user_logs_collection = db["user_logs"]
+
 answer = Blueprint("answer", __name__)
+answer_ns = Namespace("answer", description="Answer related operations", path="/")
+api.add_namespace(answer_ns)
 
 gpt_model = ""
 # to have some kind of default behaviour
@@ -74,27 +83,29 @@ def run_async_chain(chain, question, chat_history):
 
 def get_data_from_api_key(api_key):
     data = api_key_collection.find_one({"key": api_key})
-    
     # # Raise custom exception if the API key is not found
     if data is None:
         raise Exception("Invalid API Key, please generate new key", 401)
+
+    if "retriever" not in data:
+        data["retriever"] = None
+
+    if "source" in data and isinstance(data["source"], DBRef):
+        source_doc = db.dereference(data["source"])
+        data["source"] = str(source_doc["_id"])
+        if "retriever" in source_doc:
+            data["retriever"] = source_doc["retriever"]
+    else:
+        data["source"] = {}
     return data
 
 
-def get_vectorstore(data):
-    if "active_docs" in data:
-        if data["active_docs"].split("/")[0] == "default":
-            vectorstore = ""
-        elif data["active_docs"].split("/")[0] == "local":
-            vectorstore = "indexes/" + data["active_docs"]
-        else:
-            vectorstore = "vectors/" + data["active_docs"]
-        if data["active_docs"] == "default":
-            vectorstore = ""
-    else:
-        vectorstore = ""
-    vectorstore = os.path.join("application", vectorstore)
-    return vectorstore
+def get_retriever(source_id: str):
+    doc = sources_collection.find_one({"_id": ObjectId(source_id)})
+    if doc is None:
+        raise Exception("Source document does not exist", 404)
+    retriever_name = None if "retriever" not in doc else doc["retriever"]
+    return retriever_name
 
 
 def is_azure_configured():
@@ -129,10 +140,10 @@ def save_conversation(conversation_id, question, response, source_log_docs, llm)
                 "content": "Summarise following conversation in no more than 3 "
                 "words, respond ONLY with the summary, use the same "
                 "language as the system \n\nUser: "
-                +question
-                +"\n\n"
-                +"AI: "
-                +response,
+                + question
+                + "\n\n"
+                + "AI: "
+                + response,
             },
             {
                 "role": "user",
@@ -172,12 +183,21 @@ def get_prompt(prompt_id):
     return prompt
 
 
-def complete_stream(question, retriever, conversation_id, user_api_key):
+def complete_stream(
+    question, retriever, conversation_id, user_api_key, isNoneDoc=False
+):
 
     try:
         response_full = ""
         source_log_docs = []
         answer = retriever.gen()
+        sources = retriever.search()
+        for source in sources:
+            if "text" in source:
+                source["text"] = source["text"][:100].strip() + "..."
+        if len(sources) > 0:
+            data = json.dumps({"type": "source", "source": sources})
+            yield f"data: {data}\n\n"
         for line in answer:
             if "answer" in line:
                 response_full += str(line["answer"])
@@ -186,254 +206,413 @@ def complete_stream(question, retriever, conversation_id, user_api_key):
             elif "source" in line:
                 source_log_docs.append(line["source"])
 
+        if isNoneDoc:
+            for doc in source_log_docs:
+                doc["source"] = "None"
+
         llm = LLMCreator.create_llm(
             settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
-            )
-        conversation_id = save_conversation(
-            conversation_id, question, response_full, source_log_docs, llm
         )
-        
-        # send data.type = "end" to indicate that the stream has ended as json
-        data = json.dumps({"type": "id", "id": str(conversation_id)})
-        yield f"data: {data}\n\n"
+        if user_api_key is None:
+            conversation_id = save_conversation(
+                conversation_id, question, response_full, source_log_docs, llm
+            )
+            # send data.type = "end" to indicate that the stream has ended as json
+            data = json.dumps({"type": "id", "id": str(conversation_id)})
+            yield f"data: {data}\n\n"
+
+        retriever_params = retriever.get_params()
+        user_logs_collection.insert_one(
+            {
+                "action": "stream_answer",
+                "level": "info",
+                "user": "local",
+                "api_key": user_api_key,
+                "question": question,
+                "response": response_full,
+                "sources": source_log_docs,
+                "retriever_params": retriever_params,
+                "timestamp": datetime.datetime.now(datetime.timezone.utc),
+            }
+        )
         data = json.dumps({"type": "end"})
         yield f"data: {data}\n\n"
     except Exception as e:
         print("\033[91merr", str(e), file=sys.stderr)
-        data = json.dumps({"type": "error","error":"Please try again later. We apologize for any inconvenience.",
-          "error_exception": str(e)})
+        data = json.dumps(
+            {
+                "type": "error",
+                "error": "Please try again later. We apologize for any inconvenience.",
+                "error_exception": str(e),
+            }
+        )
         yield f"data: {data}\n\n"
-        return 
+        return
 
-@answer.route("/stream", methods=["POST"])
-def stream():
-   try:
-    data = request.get_json()
-    # get parameter from url question
-    question = data["question"]
-    if "history" not in data:
-        history = []
-    else:
-        history = data["history"]
-        history = json.loads(history)
-    if "conversation_id" not in data:
-        conversation_id = None
-    else:
-        conversation_id = data["conversation_id"]
-    if "prompt_id" in data:
-        prompt_id = data["prompt_id"]
-    else:
-        prompt_id = "default"
-    if "selectedDocs" in data and data["selectedDocs"] is None:
-        chunks = 0
-    elif "chunks" in data:
-        chunks = int(data["chunks"])
-    else:
-        chunks = 2
-    if "token_limit" in data:
-        token_limit = data["token_limit"]
-    else:
-        token_limit = settings.DEFAULT_MAX_HISTORY
 
-    # check if active_docs or api_key is set
-
-    if "api_key" in data:
-        data_key = get_data_from_api_key(data["api_key"])
-        chunks = int(data_key["chunks"])
-        prompt_id = data_key["prompt_id"]
-        source = {"active_docs": data_key["source"]}
-        user_api_key = data["api_key"]
-    elif "active_docs" in data:
-        source = {"active_docs": data["active_docs"]}
-        user_api_key = None
-    else:
-        source = {}
-        user_api_key = None
-
-    if (
-        source["active_docs"].split("/")[0] == "default"
-        or source["active_docs"].split("/")[0] == "local"
-    ):
-        retriever_name = "classic"
-    else:
-        retriever_name = source["active_docs"]
-
-    prompt = get_prompt(prompt_id)
-
-    retriever = RetrieverCreator.create_retriever(
-        retriever_name,
-        question=question,
-        source=source,
-        chat_history=history,
-        prompt=prompt,
-        chunks=chunks,
-        token_limit=token_limit,
-        gpt_model=gpt_model,
-        user_api_key=user_api_key,
+@answer_ns.route("/stream")
+class Stream(Resource):
+    stream_model = api.model(
+        "StreamModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
+            ),
+            "history": fields.List(
+                fields.String, required=False, description="Chat history"
+            ),
+            "conversation_id": fields.String(
+                required=False, description="Conversation ID"
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "selectedDocs": fields.String(
+                required=False, description="Selected documents"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
     )
 
-    return Response(
-        complete_stream(
-            question=question,
-            retriever=retriever,
-            conversation_id=conversation_id,
-            user_api_key=user_api_key,
-        ),
-        mimetype="text/event-stream",
-    )
-    
-   except ValueError:
-       message = "Malformed request body"
-       print("\033[91merr", str(message), file=sys.stderr)
-       return Response(
-        error_stream_generate(message),
-        status=400,
-        mimetype="text/event-stream",
-    )
-   except Exception as e:
-        print("\033[91merr", str(e), file=sys.stderr)
-        message = e.args[0]
-        status_code = 400
-        # # Custom exceptions with two arguments, index 1 as status code
-        if(len(e.args) >= 2):
-            status_code = e.args[1]
-        return Response(
-        error_stream_generate(message),
-        status=status_code,
-        mimetype="text/event-stream",
-    )
+    @api.expect(stream_model)
+    @api.doc(description="Stream a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            history = data.get("history", [])
+            history = json.loads(history)
+            conversation_id = data.get("conversation_id")
+            prompt_id = data.get("prompt_id", "default")
+            if "selectedDocs" in data and data["selectedDocs"] is None:
+                chunks = 0
+            else:
+                chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                prompt_id = data_key.get("prompt_id", "default")
+                source = {"active_docs": data_key.get("source")}
+                retriever_name = data_key.get("retriever", retriever_name)
+                user_api_key = data["api_key"]
+
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                retriever_name = get_retriever(data["active_docs"]) or retriever_name
+                user_api_key = None
+
+            else:
+                source = {}
+                user_api_key = None
+
+            current_app.logger.info(
+                f"/stream - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            prompt = get_prompt(prompt_id)
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=history,
+                prompt=prompt,
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            return Response(
+                complete_stream(
+                    question=question,
+                    retriever=retriever,
+                    conversation_id=conversation_id,
+                    user_api_key=user_api_key,
+                    isNoneDoc=data.get("isNoneDoc"),
+                ),
+                mimetype="text/event-stream",
+            )
+
+        except ValueError:
+            message = "Malformed request body"
+            print("\033[91merr", str(message), file=sys.stderr)
+            return Response(
+                error_stream_generate(message),
+                status=400,
+                mimetype="text/event-stream",
+            )
+        except Exception as e:
+            current_app.logger.error(
+                f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            message = e.args[0]
+            status_code = 400
+            # Custom exceptions with two arguments, index 1 as status code
+            if len(e.args) >= 2:
+                status_code = e.args[1]
+            return Response(
+                error_stream_generate(message),
+                status=status_code,
+                mimetype="text/event-stream",
+            )
+
+
 def error_stream_generate(err_response):
-            data = json.dumps({"type": "error", "error":err_response})
-            yield f"data: {data}\n\n"
-
-@answer.route("/api/answer", methods=["POST"])
-def api_answer():
-    data = request.get_json()
-    question = data["question"]
-    if "history" not in data:
-        history = []
-    else:
-        history = data["history"]
-    if "conversation_id" not in data:
-        conversation_id = None
-    else:
-        conversation_id = data["conversation_id"]
-    print("-" * 5)
-    if "prompt_id" in data:
-        prompt_id = data["prompt_id"]
-    else:
-        prompt_id = "default"
-    if "chunks" in data:
-        chunks = int(data["chunks"])
-    else:
-        chunks = 2
-    if "token_limit" in data:
-        token_limit = data["token_limit"]
-    else:
-        token_limit = settings.DEFAULT_MAX_HISTORY
-
-    # use try and except  to check for exception
-    try:
-        # check if the vectorstore is set
-        if "api_key" in data:
-            data_key = get_data_from_api_key(data["api_key"])
-            chunks = int(data_key["chunks"])
-            prompt_id = data_key["prompt_id"]
-            source = {"active_docs": data_key["source"]}
-            user_api_key = data["api_key"]
-        else:
-            source = data
-            user_api_key = None
-
-        if (
-            source["active_docs"].split("/")[0] == "default"
-            or source["active_docs"].split("/")[0] == "local"
-        ):
-            retriever_name = "classic"
-        else:
-            retriever_name = source["active_docs"]
-
-        prompt = get_prompt(prompt_id)
-
-        retriever = RetrieverCreator.create_retriever(
-            retriever_name,
-            question=question,
-            source=source,
-            chat_history=history,
-            prompt=prompt,
-            chunks=chunks,
-            token_limit=token_limit,
-            gpt_model=gpt_model,
-            user_api_key=user_api_key,
-        )
-        source_log_docs = []
-        response_full = ""
-        for line in retriever.gen():
-            if "source" in line:
-                source_log_docs.append(line["source"])
-            elif "answer" in line:
-                response_full += line["answer"]
-
-        llm = LLMCreator.create_llm(
-            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
-        )
-
-        result = {"answer": response_full, "sources": source_log_docs}
-        result["conversation_id"] = save_conversation(
-            conversation_id, question, response_full, source_log_docs, llm
-        )
-
-        return result
-    except Exception as e:
-        # print whole traceback
-        traceback.print_exc()
-        print(str(e))
-        return bad_request(500, str(e))
+    data = json.dumps({"type": "error", "error": err_response})
+    yield f"data: {data}\n\n"
 
 
-@answer.route("/api/search", methods=["POST"])
-def api_search():
-    data = request.get_json()
-    # get parameter from url question
-    question = data["question"]
-    if "chunks" in data:
-        chunks = int(data["chunks"])
-    else:
-        chunks = 2
-    if "api_key" in data:
-        data_key = get_data_from_api_key(data["api_key"])
-        chunks = int(data_key["chunks"])
-        source = {"active_docs": data_key["source"]}
-        user_api_key = data["api_key"]
-    elif "active_docs" in data:
-        source = {"active_docs": data["active_docs"]}
-        user_api_key = None
-    else:
-        source = {}
-        user_api_key = None
-
-    if (
-        source["active_docs"].split("/")[0] == "default"
-        or source["active_docs"].split("/")[0] == "local"
-    ):
-        retriever_name = "classic"
-    else:
-        retriever_name = source["active_docs"]
-    if "token_limit" in data:
-        token_limit = data["token_limit"]
-    else:
-        token_limit = settings.DEFAULT_MAX_HISTORY
-
-    retriever = RetrieverCreator.create_retriever(
-        retriever_name,
-        question=question,
-        source=source,
-        chat_history=[],
-        prompt="default",
-        chunks=chunks,
-        token_limit=token_limit,
-        gpt_model=gpt_model,
-        user_api_key=user_api_key,
+@answer_ns.route("/api/answer")
+class Answer(Resource):
+    answer_model = api.model(
+        "AnswerModel",
+        {
+            "question": fields.String(
+                required=True, description="The question to answer"
+            ),
+            "history": fields.List(
+                fields.String, required=False, description="Conversation history"
+            ),
+            "conversation_id": fields.String(
+                required=False, description="Conversation ID"
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
     )
-    docs = retriever.search()
-    return docs
+
+    @api.expect(answer_model)
+    @api.doc(description="Provide an answer based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            history = data.get("history", [])
+            conversation_id = data.get("conversation_id")
+            prompt_id = data.get("prompt_id", "default")
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                prompt_id = data_key.get("prompt_id", "default")
+                source = {"active_docs": data_key.get("source")}
+                retriever_name = data_key.get("retriever", retriever_name)
+                user_api_key = data["api_key"]
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                retriever_name = get_retriever(data["active_docs"]) or retriever_name
+                user_api_key = None
+            else:
+                source = {}
+                user_api_key = None
+
+            prompt = get_prompt(prompt_id)
+
+            current_app.logger.info(
+                f"/api/answer - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=history,
+                prompt=prompt,
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            source_log_docs = []
+            response_full = ""
+            for line in retriever.gen():
+                if "source" in line:
+                    source_log_docs.append(line["source"])
+                elif "answer" in line:
+                    response_full += line["answer"]
+
+            if data.get("isNoneDoc"):
+                for doc in source_log_docs:
+                    doc["source"] = "None"
+
+            llm = LLMCreator.create_llm(
+                settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
+            )
+
+            result = {"answer": response_full, "sources": source_log_docs}
+            result["conversation_id"] = str(
+                save_conversation(
+                    conversation_id, question, response_full, source_log_docs, llm
+                )
+            )
+            retriever_params = retriever.get_params()
+            user_logs_collection.insert_one(
+                {
+                    "action": "api_answer",
+                    "level": "info",
+                    "user": "local",
+                    "api_key": user_api_key,
+                    "question": question,
+                    "response": response_full,
+                    "sources": source_log_docs,
+                    "retriever_params": retriever_params,
+                    "timestamp": datetime.datetime.now(datetime.timezone.utc),
+                }
+            )
+
+        except Exception as e:
+            current_app.logger.error(
+                f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return bad_request(500, str(e))
+
+        return make_response(result, 200)
+
+
+@answer_ns.route("/api/search")
+class Search(Resource):
+    search_model = api.model(
+        "SearchModel",
+        {
+            "question": fields.String(
+                required=True, description="The question to search"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "api_key": fields.String(
+                required=False, description="API key for authentication"
+            ),
+            "active_docs": fields.String(
+                required=False, description="Active documents for retrieval"
+            ),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "token_limit": fields.Integer(
+                required=False, description="Limit for tokens"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
+    )
+
+    @api.expect(search_model)
+    @api.doc(
+        description="Search for relevant documents based on the question and retriever"
+    )
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                source = {"active_docs": data_key.get("source")}
+                user_api_key = data["api_key"]
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                user_api_key = None
+            else:
+                source = {}
+                user_api_key = None
+
+            current_app.logger.info(
+                f"/api/answer - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=[],
+                prompt="default",
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            docs = retriever.search()
+            retriever_params = retriever.get_params()
+
+            user_logs_collection.insert_one(
+                {
+                    "action": "api_search",
+                    "level": "info",
+                    "user": "local",
+                    "api_key": user_api_key,
+                    "question": question,
+                    "sources": docs,
+                    "retriever_params": retriever_params,
+                    "timestamp": datetime.datetime.now(datetime.timezone.utc),
+                }
+            )
+
+            if data.get("isNoneDoc"):
+                for doc in docs:
+                    doc["source"] = "None"
+
+        except Exception as e:
+            current_app.logger.error(
+                f"/api/search - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return bad_request(500, str(e))
+
+        return make_response(docs, 200)

application/api/internal/routes.py

@@ -3,18 +3,23 @@ import datetime
 from flask import Blueprint, request, send_from_directory
 from pymongo import MongoClient
 from werkzeug.utils import secure_filename
-
+from bson.objectid import ObjectId
 
 from application.core.settings import settings
+
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+
+internal = Blueprint("internal", __name__)
 
 
-internal = Blueprint('internal', __name__)
 @internal.route("/api/download", methods=["get"])
 def download_file():
     user = secure_filename(request.args.get("user"))
@@ -24,7 +29,6 @@ def download_file():
     return send_from_directory(save_dir, filename, as_attachment=True)
 
 
-
 @internal.route("/api/upload_index", methods=["POST"])
 def upload_index_files():
     """Upload two files(index.faiss, index.pkl) to the user's folder."""
@@ -35,7 +39,13 @@ def upload_index_files():
         return {"status": "no name"}
     job_name = secure_filename(request.form["name"])
     tokens = secure_filename(request.form["tokens"])
-    save_dir = os.path.join(current_dir, "indexes", user, job_name)
+    retriever = secure_filename(request.form["retriever"])
+    id = secure_filename(request.form["id"])
+    type = secure_filename(request.form["type"])
+    remote_data = request.form["remote_data"] if "remote_data" in request.form else None
+    sync_frequency = secure_filename(request.form["sync_frequency"]) if "sync_frequency" in request.form else None
+
+    save_dir = os.path.join(current_dir, "indexes", str(id))
     if settings.VECTOR_STORE == "faiss":
         if "file_faiss" not in request.files:
             print("No file part")
@@ -50,22 +60,45 @@ def upload_index_files():
         if file_pkl.filename == "":
             return {"status": "no file name"}
         # saves index files
-        
+
         if not os.path.exists(save_dir):
             os.makedirs(save_dir)
         file_faiss.save(os.path.join(save_dir, "index.faiss"))
         file_pkl.save(os.path.join(save_dir, "index.pkl"))
-    # create entry in vectors_collection
-    vectors_collection.insert_one(
-        {
-            "user": user,
-            "name": job_name,
-            "language": job_name,
-            "location": save_dir,
-            "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
-            "model": settings.EMBEDDINGS_NAME,
-            "type": "local",
-            "tokens": tokens
-        }
-    )
-    return {"status": "ok"}
+
+    existing_entry = sources_collection.find_one({"_id": ObjectId(id)})
+    if existing_entry:
+        sources_collection.update_one(
+            {"_id": ObjectId(id)},
+            {
+                "$set": {
+                    "user": user,
+                    "name": job_name,
+                    "language": job_name,
+                    "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "type": type,
+                    "tokens": tokens,
+                    "retriever": retriever,
+                    "remote_data": remote_data,
+                    "sync_frequency": sync_frequency,
+                }
+            },
+        )
+    else:
+        sources_collection.insert_one(
+            {
+                "_id": ObjectId(id),
+                "user": user,
+                "name": job_name,
+                "language": job_name,
+                "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
+                "model": settings.EMBEDDINGS_NAME,
+                "type": type,
+                "tokens": tokens,
+                "retriever": retriever,
+                "remote_data": remote_data,
+                "sync_frequency": sync_frequency,
+            }
+        )
+    return {"status": "ok"}

application/api/user/tasks.py

@@ -1,12 +1,38 @@
-from application.worker import ingest_worker, remote_worker
+from datetime import timedelta
+
 from application.celery_init import celery
+from application.worker import ingest_worker, remote_worker, sync_worker
+
 
 @celery.task(bind=True)
 def ingest(self, directory, formats, name_job, filename, user):
     resp = ingest_worker(self, directory, formats, name_job, filename, user)
     return resp
 
+
 @celery.task(bind=True)
 def ingest_remote(self, source_data, job_name, user, loader):
     resp = remote_worker(self, source_data, job_name, user, loader)
     return resp
+
+
+@celery.task(bind=True)
+def schedule_syncs(self, frequency):
+    resp = sync_worker(self, frequency)
+    return resp
+
+
+@celery.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+    sender.add_periodic_task(
+        timedelta(days=1),
+        schedule_syncs.s("daily"),
+    )
+    sender.add_periodic_task(
+        timedelta(weeks=1),
+        schedule_syncs.s("weekly"),
+    )
+    sender.add_periodic_task(
+        timedelta(days=30),
+        schedule_syncs.s("monthly"),
+    )

application/app.py

@@ -1,17 +1,23 @@
 import platform
+
 import dotenv
-from application.celery_init import celery
-from flask import Flask, request, redirect
-from application.core.settings import settings
-from application.api.user.routes import user
+from flask import Flask, redirect, request
+
 from application.api.answer.routes import answer
 from application.api.internal.routes import internal
+from application.api.user.routes import user
+from application.celery_init import celery
+from application.core.logging_config import setup_logging
+from application.core.settings import settings
+from application.extensions import api
 
 if platform.system() == "Windows":
     import pathlib
+
     pathlib.PosixPath = pathlib.WindowsPath
 
 dotenv.load_dotenv()
+setup_logging()
 
 app = Flask(__name__)
 app.register_blueprint(user)
@@ -21,16 +27,19 @@ app.config.update(
     UPLOAD_FOLDER="inputs",
     CELERY_BROKER_URL=settings.CELERY_BROKER_URL,
     CELERY_RESULT_BACKEND=settings.CELERY_RESULT_BACKEND,
-    MONGO_URI=settings.MONGO_URI
+    MONGO_URI=settings.MONGO_URI,
 )
 celery.config_from_object("application.celeryconfig")
+api.init_app(app)
+
 
 @app.route("/")
 def home():
-    if request.remote_addr in ('0.0.0.0', '127.0.0.1', 'localhost', '172.18.0.1'):
-        return redirect('http://localhost:5173')
+    if request.remote_addr in ("0.0.0.0", "127.0.0.1", "localhost", "172.18.0.1"):
+        return redirect("http://localhost:5173")
     else:
-        return 'Welcome to DocsGPT Backend!'
+        return "Welcome to DocsGPT Backend!"
+
 
 @app.after_request
 def after_request(response):
@@ -39,6 +48,6 @@ def after_request(response):
     response.headers.add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS")
     return response
 
+
 if __name__ == "__main__":
     app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)
-

application/celery_init.py

@@ -1,9 +1,15 @@
 from celery import Celery
 from application.core.settings import settings
+from celery.signals import setup_logging
 
 def make_celery(app_name=__name__):
     celery = Celery(app_name, broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND)
     celery.conf.update(settings)
     return celery
 
+@setup_logging.connect
+def config_loggers(*args, **kwargs):
+    from application.core.logging_config import setup_logging
+    setup_logging()
+
 celery = make_celery()

application/core/logging_config.py

@@ -0,0 +1,22 @@
+from logging.config import dictConfig
+
+def setup_logging():
+    dictConfig({
+        'version': 1,
+        'formatters': {
+            'default': {
+                'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+            }
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "stream": "ext://sys.stdout",
+                "formatter": "default",
+            }
+        },
+        'root': {
+            'level': 'INFO',
+            'handlers': ['console'],
+        },
+    })

application/core/settings.py

@@ -18,7 +18,7 @@ class Settings(BaseSettings):
     DEFAULT_MAX_HISTORY: int = 150
     MODEL_TOKEN_LIMITS: dict = {"gpt-3.5-turbo": 4096, "claude-2": 1e5}
     UPLOAD_FOLDER: str = "inputs"
-    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant"
+    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant" or "milvus"
     RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
 
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
@@ -29,6 +29,7 @@ class Settings(BaseSettings):
     OPENAI_API_VERSION: Optional[str] = None  # azure openai api version
     AZURE_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for answering
     AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for embeddings
+    OPENAI_BASE_URL: Optional[str] = None # openai base url for open ai compatable models
 
     # elasticsearch
     ELASTIC_CLOUD_ID: Optional[str] = None  # cloud id for elasticsearch
@@ -61,6 +62,11 @@ class Settings(BaseSettings):
     QDRANT_PATH: Optional[str] = None
     QDRANT_DISTANCE_FUNC: str = "Cosine"
 
+    # Milvus vectorstore config
+    MILVUS_COLLECTION_NAME: Optional[str] = "docsgpt"
+    MILVUS_URI: Optional[str] = "./milvus_local.db"   # milvus lite version as default
+    MILVUS_TOKEN: Optional[str] = ""
+
     BRAVE_SEARCH_API_KEY: Optional[str] = None
 
     FLASK_DEBUG_MODE: bool = False

application/extensions.py

@@ -0,0 +1,7 @@
+from flask_restx import Api
+
+api = Api(
+    version="1.0",
+    title="DocsGPT API",
+    description="API for DocsGPT",
+)

application/llm/openai.py

@@ -2,25 +2,23 @@ from application.llm.base import BaseLLM
 from application.core.settings import settings
 
 
+
 class OpenAILLM(BaseLLM):
 
     def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
-        global openai
         from openai import OpenAI
 
         super().__init__(*args, **kwargs)
-        self.client = OpenAI(
-            api_key=api_key,
-        )
+        if settings.OPENAI_BASE_URL:
+            self.client = OpenAI(
+                api_key=api_key,
+                base_url=settings.OPENAI_BASE_URL
+            )
+        else:
+            self.client = OpenAI(api_key=api_key)
         self.api_key = api_key
         self.user_api_key = user_api_key
 
-    def _get_openai(self):
-        # Import openai when needed
-        import openai
-
-        return openai
-
     def _raw_gen(
         self,
         baseself,
@@ -29,7 +27,7 @@ class OpenAILLM(BaseLLM):
         stream=False,
         engine=settings.AZURE_DEPLOYMENT_NAME,
         **kwargs
-    ):
+    ):  
         response = self.client.chat.completions.create(
             model=model, messages=messages, stream=stream, **kwargs
         )
@@ -44,7 +42,7 @@ class OpenAILLM(BaseLLM):
         stream=True,
         engine=settings.AZURE_DEPLOYMENT_NAME,
         **kwargs
-    ):
+    ):  
         response = self.client.chat.completions.create(
             model=model, messages=messages, stream=stream, **kwargs
         )
@@ -73,8 +71,3 @@ class AzureOpenAILLM(OpenAILLM):
             api_base=settings.OPENAI_API_BASE,
             deployment_name=settings.AZURE_DEPLOYMENT_NAME,
         )
-
-    def _get_openai(self):
-        openai = super()._get_openai()
-
-        return openai

application/parser/file/html_parser.py

@@ -3,7 +3,6 @@
 Contains parser for html files.
 
 """
-import re
 from pathlib import Path
 from typing import Dict, Union
 
@@ -18,66 +17,8 @@ class HTMLParser(BaseParser):
         return {}
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
-        """Parse file.
+        from langchain_community.document_loaders import BSHTMLLoader
 
-            Returns:
-            Union[str, List[str]]: a string or a List of strings.
-        """
-        try:
-            from unstructured.partition.html import partition_html
-            from unstructured.staging.base import convert_to_isd
-            from unstructured.cleaners.core import clean
-        except ImportError:
-            raise ValueError("unstructured package is required to parse HTML files.")
-
-        # Using the unstructured library to convert the html to isd format
-        # isd sample : isd = [
-        #   {"text": "My Title", "type": "Title"},
-        #   {"text": "My Narrative", "type": "NarrativeText"}
-        # ]
-        with open(file, "r", encoding="utf-8") as fp:
-            elements = partition_html(file=fp)
-            isd = convert_to_isd(elements)
-
-            # Removing non ascii charactwers from isd_el['text']
-        for isd_el in isd:
-            isd_el['text'] = isd_el['text'].encode("ascii", "ignore").decode()
-
-        # Removing all the \n characters from isd_el['text'] using regex and replace with single space
-        # Removing all the extra spaces  from isd_el['text'] using regex and replace with single space
-        for isd_el in isd:
-            isd_el['text'] = re.sub(r'\n', ' ', isd_el['text'], flags=re.MULTILINE | re.DOTALL)
-            isd_el['text'] = re.sub(r"\s{2,}", " ", isd_el['text'], flags=re.MULTILINE | re.DOTALL)
-
-        # more cleaning: extra_whitespaces, dashes, bullets, trailing_punctuation
-        for isd_el in isd:
-            clean(isd_el['text'], extra_whitespace=True, dashes=True, bullets=True, trailing_punctuation=True)
-
-        # Creating a list of all the indexes of isd_el['type'] = 'Title'
-        title_indexes = [i for i, isd_el in enumerate(isd) if isd_el['type'] == 'Title']
-
-        # Creating 'Chunks' - List of lists of strings 
-        # each list starting with isd_el['type'] = 'Title' and all the data till the next 'Title'
-        # Each Chunk can be thought of as an individual set of data, which can be sent to the model
-        # Where Each Title is grouped together with the data under it
-
-        Chunks = [[]]
-        final_chunks = list(list())
-
-        for i, isd_el in enumerate(isd):
-            if i in title_indexes:
-                Chunks.append([])
-            Chunks[-1].append(isd_el['text'])
-
-        # Removing all the chunks with sum of length of all the strings in the chunk < 25
-        # TODO: This value can be an user defined variable
-        for chunk in Chunks:
-            # sum of length of all the strings in the chunk
-            sum = 0
-            sum += len(str(chunk))
-            if sum < 25:
-                Chunks.remove(chunk)
-            else:
-                # appending all the approved chunks to final_chunks as a single string       
-                final_chunks.append(" ".join([str(item) for item in chunk]))
-        return final_chunks
+        loader = BSHTMLLoader(file)
+        data = loader.load()        
+        return data

application/parser/open_ai_func.py

@@ -1,9 +1,11 @@
 import os
 
-from application.vectorstore.vector_creator import VectorCreator
-from application.core.settings import settings
 from retry import retry
 
+from application.core.settings import settings
+
+from application.vectorstore.vector_creator import VectorCreator
+
 
 # from langchain_community.embeddings import HuggingFaceEmbeddings
 # from langchain_community.embeddings import HuggingFaceInstructEmbeddings
@@ -11,12 +13,14 @@ from retry import retry
 
 
 @retry(tries=10, delay=60)
-def store_add_texts_with_retry(store, i):
+def store_add_texts_with_retry(store, i, id):
+    # add source_id to the metadata
+    i.metadata["source_id"] = str(id)
     store.add_texts([i.page_content], metadatas=[i.metadata])
     # store_pine.add_texts([i.page_content], metadatas=[i.metadata])
 
 
-def call_openai_api(docs, folder_name, task_status):
+def call_openai_api(docs, folder_name, id, task_status):
     # Function to create a vector store from the documents and save it to disk
 
     if not os.path.exists(f"{folder_name}"):
@@ -32,15 +36,16 @@ def call_openai_api(docs, folder_name, task_status):
         store = VectorCreator.create_vectorstore(
             settings.VECTOR_STORE,
             docs_init=docs_init,
-            path=f"{folder_name}",
+            source_id=f"{folder_name}",
             embeddings_key=os.getenv("EMBEDDINGS_KEY"),
         )
     else:
         store = VectorCreator.create_vectorstore(
             settings.VECTOR_STORE,
-            path=f"{folder_name}",
+            source_id=str(id),
             embeddings_key=os.getenv("EMBEDDINGS_KEY"),
         )
+        store.delete_index()
     # Uncomment for MPNet embeddings
     # model_name = "sentence-transformers/all-mpnet-base-v2"
     # hf = HuggingFaceEmbeddings(model_name=model_name)
@@ -57,7 +62,7 @@ def call_openai_api(docs, folder_name, task_status):
             task_status.update_state(
                 state="PROGRESS", meta={"current": int((c1 / s1) * 100)}
             )
-            store_add_texts_with_retry(store, i)
+            store_add_texts_with_retry(store, i, id)
         except Exception as e:
             print(e)
             print("Error on ", i)
@@ -68,5 +73,3 @@ def call_openai_api(docs, folder_name, task_status):
         c1 += 1
     if settings.VECTOR_STORE == "faiss":
         store.save_local(f"{folder_name}")
-
-

application/parser/remote/crawler_loader.py

@@ -5,7 +5,7 @@ from application.parser.remote.base import BaseRemote
 
 class CrawlerLoader(BaseRemote):
     def __init__(self, limit=10):
-        from langchain.document_loaders import WebBaseLoader
+        from langchain_community.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader  # Initialize the document loader
         self.limit = limit  # Set the limit for the number of pages to scrape
 

application/parser/remote/sitemap_loader.py

@@ -5,7 +5,7 @@ from application.parser.remote.base import BaseRemote
 
 class SitemapLoader(BaseRemote):
     def __init__(self, limit=20):
-        from langchain.document_loaders import WebBaseLoader
+        from langchain_community.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader
         self.limit = limit  # Adding limit to control the number of URLs to process
 

application/requirements.txt

@@ -1,34 +1,86 @@
-anthropic==0.12.0
-boto3==1.34.6
+anthropic==0.34.2
+boto3==1.34.153
+beautifulsoup4==4.12.3
 celery==5.3.6
-dataclasses_json==0.6.3
+dataclasses-json==0.6.7
 docx2txt==0.8
-duckduckgo-search==5.3.0
-EbookLib==0.18
-elasticsearch==8.12.0
+duckduckgo-search==6.2.6
+ebooklib==0.18
+elastic-transport==8.15.0
+elasticsearch==8.15.1
 escodegen==1.0.11
 esprima==4.0.1
-faiss-cpu==1.7.4
-Flask==3.0.1
-gunicorn==22.0.0
-html2text==2020.1.16
+esutils==1.0.1
+Flask==3.0.3
+faiss-cpu==1.8.0.post1
+flask-restx==1.3.0
+gunicorn==23.0.0
+html2text==2024.2.26
 javalang==0.13.0
-langchain==0.1.4
-langchain-openai==0.0.5
-openapi3_parser==1.1.16
-pandas==2.2.0
-pydantic_settings==2.1.0
-pymongo==4.6.3
-PyPDF2==3.0.1
+jinja2==3.1.4
+jiter==0.5.0
+jmespath==1.0.1
+joblib==1.4.2
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-spec==0.2.4
+jsonschema-specifications==2023.7.1
+kombu==5.4.2
+langchain==0.3.0
+langchain-community==0.3.0
+langchain-core==0.3.2
+langchain-openai==0.2.0
+langchain-text-splitters==0.3.0
+langsmith==0.1.125
+lazy-object-proxy==1.10.0
+lxml==5.3.0
+markupsafe==2.1.5
+marshmallow==3.22.0
+mpmath==1.3.0
+multidict==6.1.0
+mypy-extensions==1.0.0
+networkx==3.3
+numpy==1.26.4
+openai==1.46.1
+openapi-schema-validator==0.6.2
+openapi-spec-validator==0.6.0
+openapi3-parser==1.1.18
+orjson==3.10.7
+packaging==24.1
+pandas==2.2.3
+pathable==0.4.3
+pillow==10.4.0
+portalocker==2.10.1
+prance==23.6.21.0
+primp==0.6.2
+prompt-toolkit==3.0.47
+protobuf==5.28.2
+py==1.11.0
+pydantic==2.9.2
+pydantic-core==2.23.4
+pydantic-settings==2.4.0
+pymongo==4.8.0
+pypdf2==3.0.1
+python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
-qdrant-client==1.9.0
+qdrant-client==1.11.0
 redis==5.0.1
-Requests==2.32.0
+referencing==0.30.2
+regex==2024.9.11
+requests==2.32.3
 retry==0.9.2
-sentence-transformers
-tiktoken
-torch
-tqdm==4.66.3
-transformers==4.36.2
-unstructured==0.12.2
-Werkzeug==3.0.3
+sentence-transformers==3.0.1
+tiktoken==0.7.0
+tokenizers==0.19.1
+torch==2.4.1
+tqdm==4.66.5
+transformers==4.44.2
+typing-extensions==4.12.2
+typing-inspect==0.9.0
+tzdata==2024.2
+urllib3==2.2.3
+vine==5.1.0
+wcwidth==0.2.13
+werkzeug==3.0.4
+yarl==1.11.1

application/retriever/base.py

@@ -12,3 +12,7 @@ class BaseRetriever(ABC):
     @abstractmethod
     def search(self, *args, **kwargs):
         pass
+
+    @abstractmethod
+    def get_params(self):
+        pass

application/retriever/brave_search.py

@@ -2,7 +2,7 @@ import json
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.llm.llm_creator import LLMCreator
-from application.utils import count_tokens
+from application.utils import num_tokens_from_string
 from langchain_community.tools import BraveSearch
 
 
@@ -78,7 +78,7 @@ class BraveRetSearch(BaseRetriever):
             self.chat_history.reverse()
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
+                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
                         i["response"]
                     )
                     if tokens_current_history + tokens_batch < self.token_limit:
@@ -101,3 +101,15 @@ class BraveRetSearch(BaseRetriever):
 
     def search(self):
         return self._get_data()
+
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/classic_rag.py

@@ -1,10 +1,9 @@
-import os
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.vectorstore.vector_creator import VectorCreator
 from application.llm.llm_creator import LLMCreator
 
-from application.utils import count_tokens
+from application.utils import num_tokens_from_string
 
 
 class ClassicRAG(BaseRetriever):
@@ -21,7 +20,7 @@ class ClassicRAG(BaseRetriever):
         user_api_key=None,
     ):
         self.question = question
-        self.vectorstore = self._get_vectorstore(source=source)
+        self.vectorstore = source['active_docs'] if 'active_docs' in source else None
         self.chat_history = chat_history
         self.prompt = prompt
         self.chunks = chunks
@@ -38,21 +37,6 @@ class ClassicRAG(BaseRetriever):
         )
         self.user_api_key = user_api_key
 
-    def _get_vectorstore(self, source):
-        if "active_docs" in source:
-            if source["active_docs"].split("/")[0] == "default":
-                vectorstore = ""
-            elif source["active_docs"].split("/")[0] == "local":
-                vectorstore = "indexes/" + source["active_docs"]
-            else:
-                vectorstore = "vectors/" + source["active_docs"]
-            if source["active_docs"] == "default":
-                vectorstore = ""
-        else:
-            vectorstore = ""
-        vectorstore = os.path.join("application", vectorstore)
-        return vectorstore
-
     def _get_data(self):
         if self.chunks == 0:
             docs = []
@@ -61,13 +45,12 @@ class ClassicRAG(BaseRetriever):
                 settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
             )
             docs_temp = docsearch.search(self.question, k=self.chunks)
+            print(docs_temp)
             docs = [
                 {
-                    "title": (
-                        i.metadata["title"].split("/")[-1]
-                        if i.metadata
-                        else i.page_content
-                    ),
+                    "title": i.metadata.get(
+                        "title", i.metadata.get("post_title", i.page_content)
+                    ).split("/")[-1],
                     "text": i.page_content,
                     "source": (
                         i.metadata.get("source")
@@ -98,7 +81,7 @@ class ClassicRAG(BaseRetriever):
             self.chat_history.reverse()
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
+                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
                         i["response"]
                     )
                     if tokens_current_history + tokens_batch < self.token_limit:
@@ -121,3 +104,15 @@ class ClassicRAG(BaseRetriever):
 
     def search(self):
         return self._get_data()
+    
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.vectorstore,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/duckduck_search.py

@@ -1,7 +1,7 @@
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.llm.llm_creator import LLMCreator
-from application.utils import count_tokens
+from application.utils import num_tokens_from_string
 from langchain_community.tools import DuckDuckGoSearchResults
 from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
 
@@ -95,7 +95,7 @@ class DuckDuckSearch(BaseRetriever):
             self.chat_history.reverse()
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
+                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
                         i["response"]
                     )
                     if tokens_current_history + tokens_batch < self.token_limit:
@@ -118,3 +118,15 @@ class DuckDuckSearch(BaseRetriever):
 
     def search(self):
         return self._get_data()
+    
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/retriever_creator.py

@@ -5,15 +5,16 @@ from application.retriever.brave_search import BraveRetSearch
 
 
 class RetrieverCreator:
-    retievers = {
+    retrievers = {
         'classic': ClassicRAG,
         'duckduck_search': DuckDuckSearch,
-        'brave_search': BraveRetSearch
+        'brave_search': BraveRetSearch,
+        'default': ClassicRAG
     }
 
     @classmethod
     def create_retriever(cls, type, *args, **kwargs):
-        retiever_class = cls.retievers.get(type.lower())
+        retiever_class = cls.retrievers.get(type.lower())
         if not retiever_class:
             raise ValueError(f"No retievers class found for type {type}")
         return retiever_class(*args, **kwargs)

application/usage.py

@@ -2,7 +2,7 @@ import sys
 from pymongo import MongoClient
 from datetime import datetime
 from application.core.settings import settings
-from application.utils import count_tokens
+from application.utils import num_tokens_from_string
 
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
@@ -24,9 +24,9 @@ def update_token_usage(user_api_key, token_usage):
 def gen_token_usage(func):
     def wrapper(self, model, messages, stream, **kwargs):
         for message in messages:
-            self.token_usage["prompt_tokens"] += count_tokens(message["content"])
+            self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
         result = func(self, model, messages, stream, **kwargs)
-        self.token_usage["generated_tokens"] += count_tokens(result)
+        self.token_usage["generated_tokens"] += num_tokens_from_string(result)
         update_token_usage(self.user_api_key, self.token_usage)
         return result
 
@@ -36,14 +36,14 @@ def gen_token_usage(func):
 def stream_token_usage(func):
     def wrapper(self, model, messages, stream, **kwargs):
         for message in messages:
-            self.token_usage["prompt_tokens"] += count_tokens(message["content"])
+            self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
         batch = []
         result = func(self, model, messages, stream, **kwargs)
         for r in result:
             batch.append(r)
             yield r
         for line in batch:
-            self.token_usage["generated_tokens"] += count_tokens(line)
+            self.token_usage["generated_tokens"] += num_tokens_from_string(line)
         update_token_usage(self.user_api_key, self.token_usage)
 
     return wrapper

application/utils.py

@@ -1,6 +1,41 @@
-from transformers import GPT2TokenizerFast
+import tiktoken
+from flask import jsonify, make_response
 
-tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
-tokenizer.model_max_length = 100000
-def count_tokens(string):
-    return len(tokenizer(string)['input_ids'])
+_encoding = None
+
+
+def get_encoding():
+    global _encoding
+    if _encoding is None:
+        _encoding = tiktoken.get_encoding("cl100k_base")
+    return _encoding
+
+
+def num_tokens_from_string(string: str) -> int:
+    encoding = get_encoding()
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
+
+
+def count_tokens_docs(docs):
+    docs_content = ""
+    for doc in docs:
+        docs_content += doc.page_content
+
+    tokens = num_tokens_from_string(docs_content)
+    return tokens
+
+
+def check_required_fields(data, required_fields):
+    missing_fields = [field for field in required_fields if field not in data]
+    if missing_fields:
+        return make_response(
+            jsonify(
+                {
+                    "success": False,
+                    "message": f"Missing fields: {', '.join(missing_fields)}",
+                }
+            ),
+            400,
+        )
+    return None

application/vectorstore/base.py

@@ -1,13 +1,30 @@
 from abc import ABC, abstractmethod
 import os
-from langchain_community.embeddings import (
-    HuggingFaceEmbeddings,
-    CohereEmbeddings,
-    HuggingFaceInstructEmbeddings,
-)
+from sentence_transformers import SentenceTransformer
 from langchain_openai import OpenAIEmbeddings
 from application.core.settings import settings
 
+class EmbeddingsWrapper:
+    def __init__(self, model_name, *args, **kwargs):
+        self.model = SentenceTransformer(model_name, config_kwargs={'allow_dangerous_deserialization': True}, *args, **kwargs)
+        self.dimension = self.model.get_sentence_embedding_dimension()
+
+    def embed_query(self, query: str):
+        return self.model.encode(query).tolist()
+    
+    def embed_documents(self, documents: list):
+        return self.model.encode(documents).tolist()
+    
+    def __call__(self, text):
+        if isinstance(text, str):
+            return self.embed_query(text)
+        elif isinstance(text, list):
+            return self.embed_documents(text)
+        else:
+            raise ValueError("Input must be a string or a list of strings")
+
+
+
 class EmbeddingsSingleton:
     _instances = {}
 
@@ -23,16 +40,15 @@ class EmbeddingsSingleton:
     def _create_instance(embeddings_name, *args, **kwargs):
         embeddings_factory = {
             "openai_text-embedding-ada-002": OpenAIEmbeddings,
-            "huggingface_sentence-transformers/all-mpnet-base-v2": HuggingFaceEmbeddings,
-            "huggingface_sentence-transformers-all-mpnet-base-v2": HuggingFaceEmbeddings,
-            "huggingface_hkunlp/instructor-large": HuggingFaceInstructEmbeddings,
-            "cohere_medium": CohereEmbeddings
+            "huggingface_sentence-transformers/all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
+            "huggingface_sentence-transformers-all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
+            "huggingface_hkunlp/instructor-large": lambda: EmbeddingsWrapper("hkunlp/instructor-large"),
         }
 
-        if embeddings_name not in embeddings_factory:
-            raise ValueError(f"Invalid embeddings_name: {embeddings_name}")
-
-        return embeddings_factory[embeddings_name](*args, **kwargs)
+        if embeddings_name in embeddings_factory:
+            return embeddings_factory[embeddings_name](*args, **kwargs)
+        else:
+            return EmbeddingsWrapper(embeddings_name, *args, **kwargs)
 
 class BaseVectorStore(ABC):
     def __init__(self):
@@ -58,22 +74,14 @@ class BaseVectorStore(ABC):
                     embeddings_name,
                     openai_api_key=embeddings_key
                 )
-        elif embeddings_name == "cohere_medium":
-            embedding_instance = EmbeddingsSingleton.get_instance(
-                embeddings_name,
-                cohere_api_key=embeddings_key
-            )
         elif embeddings_name == "huggingface_sentence-transformers/all-mpnet-base-v2":
             if os.path.exists("./model/all-mpnet-base-v2"):
                 embedding_instance = EmbeddingsSingleton.get_instance(
-                    embeddings_name,
-                    model_name="./model/all-mpnet-base-v2",
-                    model_kwargs={"device": "cpu"}
+                    embeddings_name="./model/all-mpnet-base-v2",
                 )
             else:
                 embedding_instance = EmbeddingsSingleton.get_instance(
                     embeddings_name,
-                    model_kwargs={"device": "cpu"}
                 )
         else:
             embedding_instance = EmbeddingsSingleton.get_instance(embeddings_name)

application/vectorstore/elasticsearch.py

@@ -9,9 +9,9 @@ import elasticsearch
 class ElasticsearchStore(BaseVectorStore):
     _es_connection = None  # Class attribute to hold the Elasticsearch connection
 
-    def __init__(self, path, embeddings_key, index_name=settings.ELASTIC_INDEX):
+    def __init__(self, source_id, embeddings_key, index_name=settings.ELASTIC_INDEX):
         super().__init__()
-        self.path = path.replace("application/indexes/", "").rstrip("/")
+        self.source_id = source_id.replace("application/indexes/", "").rstrip("/")
         self.embeddings_key = embeddings_key
         self.index_name = index_name
         
@@ -81,7 +81,7 @@ class ElasticsearchStore(BaseVectorStore):
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
         vector = embeddings.embed_query(question)
         knn = {
-            "filter": [{"match": {"metadata.store.keyword": self.path}}],
+            "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
             "field": "vector",
             "k": k,
             "num_candidates": 100,
@@ -100,7 +100,7 @@ class ElasticsearchStore(BaseVectorStore):
                             }
                         }
                     ],
-                    "filter": [{"match": {"metadata.store.keyword": self.path}}],
+                    "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
                 }
             },
             "rank": {"rrf": {}},
@@ -209,5 +209,4 @@ class ElasticsearchStore(BaseVectorStore):
 
     def delete_index(self):
         self._es_connection.delete_by_query(index=self.index_name, query={"match": {
-                                      "metadata.store.keyword": self.path}},)
-
+                                      "metadata.source_id.keyword": self.source_id}},)

application/vectorstore/faiss.py

@@ -1,12 +1,22 @@
 from langchain_community.vectorstores import FAISS
 from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
+import os
+
+def get_vectorstore(path):
+    if path:
+        vectorstore = "indexes/"+path
+        vectorstore = os.path.join("application", vectorstore)
+    else:
+        vectorstore = os.path.join("application")
+
+    return vectorstore
 
 class FaissStore(BaseVectorStore):
 
-    def __init__(self, path, embeddings_key, docs_init=None):
+    def __init__(self, source_id, embeddings_key, docs_init=None):
         super().__init__()
-        self.path = path
+        self.path = get_vectorstore(source_id)
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
         if docs_init:
             self.docsearch = FAISS.from_documents(
@@ -14,7 +24,8 @@ class FaissStore(BaseVectorStore):
             )
         else:
             self.docsearch = FAISS.load_local(
-                self.path, embeddings
+                self.path, embeddings, 
+                allow_dangerous_deserialization=True
             )
         self.assert_embedding_dimensions(embeddings)
 
@@ -37,10 +48,10 @@ class FaissStore(BaseVectorStore):
         """
         if settings.EMBEDDINGS_NAME == "huggingface_sentence-transformers/all-mpnet-base-v2":
             try:
-                word_embedding_dimension = embeddings.client[1].word_embedding_dimension
+                word_embedding_dimension = embeddings.dimension
             except AttributeError as e:
-                raise AttributeError("word_embedding_dimension not found in embeddings.client[1]") from e
+                raise AttributeError("'dimension' attribute not found in embeddings instance. Make sure the embeddings object is properly initialized.") from e
             docsearch_index_dimension = self.docsearch.index.d
             if word_embedding_dimension != docsearch_index_dimension:
-                raise ValueError(f"word_embedding_dimension ({word_embedding_dimension}) " +
-                                 f"!= docsearch_index_word_embedding_dimension ({docsearch_index_dimension})")
+                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) " +
+                                 f"!= docsearch index dimension ({docsearch_index_dimension})")

application/vectorstore/milvus.py

@@ -0,0 +1,37 @@
+from typing import List, Optional
+from uuid import uuid4
+
+
+from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
+
+
+class MilvusStore(BaseVectorStore):
+    def __init__(self, path: str = "", embeddings_key: str = "embeddings"):
+        super().__init__()
+        from langchain_milvus import Milvus
+
+        connection_args = {
+            "uri": settings.MILVUS_URI,
+            "token": settings.MILVUS_TOKEN,
+        }
+        self._docsearch = Milvus(
+            embedding_function=self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key),
+            collection_name=settings.MILVUS_COLLECTION_NAME,
+            connection_args=connection_args,
+        )
+        self._path = path
+
+    def search(self, question, k=2, *args, **kwargs):
+        return self._docsearch.similarity_search(query=question, k=k, filter={"path": self._path} *args, **kwargs)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]], *args, **kwargs):
+        ids = [str(uuid4()) for _ in range(len(texts))]
+
+        return self._docsearch.add_texts(texts=texts, metadatas=metadatas, ids=ids, *args, **kwargs)
+
+    def save_local(self, *args, **kwargs):
+        pass
+
+    def delete_index(self, *args, **kwargs):
+        pass

application/vectorstore/mongodb.py

@@ -1,11 +1,12 @@
-from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
 from application.vectorstore.document_class import Document
 
+
 class MongoDBVectorStore(BaseVectorStore):
     def __init__(
         self,
-        path: str = "",
+        source_id: str = "",
         embeddings_key: str = "embeddings",
         collection: str = "documents",
         index_name: str = "vector_search_index",
@@ -18,7 +19,7 @@ class MongoDBVectorStore(BaseVectorStore):
         self._embedding_key = embedding_key
         self._embeddings_key = embeddings_key
         self._mongo_uri = settings.MONGO_URI
-        self._path = path.replace("application/indexes/", "").rstrip("/")
+        self._source_id = source_id.replace("application/indexes/", "").rstrip("/")
         self._embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
 
         try:
@@ -33,27 +34,24 @@ class MongoDBVectorStore(BaseVectorStore):
         self._database = self._client[database]
         self._collection = self._database[collection]
 
-        
     def search(self, question, k=2, *args, **kwargs):
         query_vector = self._embedding.embed_query(question)
 
         pipeline = [
             {
                 "$vectorSearch": {
-                    "queryVector": query_vector, 
+                    "queryVector": query_vector,
                     "path": self._embedding_key,
-                    "limit": k, 
-                    "numCandidates": k * 10, 
+                    "limit": k,
+                    "numCandidates": k * 10,
                     "index": self._index_name,
-                    "filter": {
-                        "store": {"$eq": self._path}
-                    }
+                    "filter": {"source_id": {"$eq": self._source_id}},
                 }
             }
         ]
 
         cursor = self._collection.aggregate(pipeline)
-        
+
         results = []
         for doc in cursor:
             text = doc[self._text_key]
@@ -63,30 +61,32 @@ class MongoDBVectorStore(BaseVectorStore):
             metadata = doc
             results.append(Document(text, metadata))
         return results
-    
+
     def _insert_texts(self, texts, metadatas):
         if not texts:
             return []
         embeddings = self._embedding.embed_documents(texts)
+
         to_insert = [
             {self._text_key: t, self._embedding_key: embedding, **m}
             for t, m, embedding in zip(texts, metadatas, embeddings)
         ]
-        # insert the documents in MongoDB Atlas
+
         insert_result = self._collection.insert_many(to_insert)
         return insert_result.inserted_ids
-    
-    def add_texts(self,
+
+    def add_texts(
+        self,
         texts,
-        metadatas = None,
-        ids = None,
-        refresh_indices = True,
-        create_index_if_not_exists = True,
-        bulk_kwargs = None,
-        **kwargs,):
+        metadatas=None,
+        ids=None,
+        refresh_indices=True,
+        create_index_if_not_exists=True,
+        bulk_kwargs=None,
+        **kwargs,
+    ):
 
-
-        #dims = self._embedding.client[1].word_embedding_dimension
+        # dims = self._embedding.client[1].word_embedding_dimension
         # # check if index exists
         # if create_index_if_not_exists:
         #     # check if index exists
@@ -121,6 +121,6 @@ class MongoDBVectorStore(BaseVectorStore):
         if texts_batch:
             result_ids.extend(self._insert_texts(texts_batch, metadatas_batch))
         return result_ids
-    
+
     def delete_index(self, *args, **kwargs):
-        self._collection.delete_many({"store": self._path})
+        self._collection.delete_many({"source_id": self._source_id})

application/vectorstore/qdrant.py

@@ -5,12 +5,12 @@ from qdrant_client import models
 
 
 class QdrantStore(BaseVectorStore):
-    def __init__(self, path: str = "", embeddings_key: str = "embeddings"):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
         self._filter = models.Filter(
             must=[
                 models.FieldCondition(
-                    key="metadata.store",
-                    match=models.MatchValue(value=path.replace("application/indexes/", "").rstrip("/")),
+                    key="metadata.source_id",
+                    match=models.MatchValue(value=source_id.replace("application/indexes/", "").rstrip("/")),
                 )
             ]
         )

application/vectorstore/vector_creator.py

@@ -1,5 +1,6 @@
 from application.vectorstore.faiss import FaissStore
 from application.vectorstore.elasticsearch import ElasticsearchStore
+from application.vectorstore.milvus import MilvusStore
 from application.vectorstore.mongodb import MongoDBVectorStore
 from application.vectorstore.qdrant import QdrantStore
 
@@ -10,6 +11,7 @@ class VectorCreator:
         "elasticsearch": ElasticsearchStore,
         "mongodb": MongoDBVectorStore,
         "qdrant": QdrantStore,
+        "milvus": MilvusStore,
     }
 
     @classmethod

application/worker.py

@@ -1,23 +1,31 @@
+import logging
 import os
 import shutil
 import string
 import zipfile
-import tiktoken
+from collections import Counter
 from urllib.parse import urljoin
 
 import requests
+from bson.objectid import ObjectId
+from pymongo import MongoClient
 
 from application.core.settings import settings
 from application.parser.file.bulk import SimpleDirectoryReader
-from application.parser.remote.remote_creator import RemoteCreator
 from application.parser.open_ai_func import call_openai_api
+from application.parser.remote.remote_creator import RemoteCreator
 from application.parser.schema.base import Document
 from application.parser.token_func import group_split
+from application.utils import count_tokens_docs
+
+mongo = MongoClient(settings.MONGO_URI)
+db = mongo["docsgpt"]
+sources_collection = db["sources"]
+
 
 # Define a function to extract metadata from a given filename.
 def metadata_from_filename(title):
-    store = "/".join(title.split("/")[1:3])
-    return {"title": title, "store": store}
+    return {"title": title}
 
 
 # Define a function to generate a random string of a given length.
@@ -41,7 +49,7 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
         max_depth (int): Maximum allowed depth of recursion to prevent infinite loops.
     """
     if current_depth > max_depth:
-        print(f"Reached maximum recursion depth of {max_depth}")
+        logging.warning(f"Reached maximum recursion depth of {max_depth}")
         return
 
     with zipfile.ZipFile(zip_path, "r") as zip_ref:
@@ -58,7 +66,9 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
 
 
 # Define the main function for ingesting and processing documents.
-def ingest_worker(self, directory, formats, name_job, filename, user):
+def ingest_worker(
+    self, directory, formats, name_job, filename, user, retriever="classic"
+):
     """
     Ingest and process documents.
 
@@ -69,6 +79,7 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         name_job (str): Name of the job for this ingestion task.
         filename (str): Name of the file to be ingested.
         user (str): Identifier for the user initiating the ingestion.
+        retriever (str): Type of retriever to use for processing the documents.
 
     Returns:
         dict: Information about the completed ingestion task, including input parameters and a "limited" flag.
@@ -88,16 +99,13 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
     max_tokens = 1250
     recursion_depth = 2
     full_path = os.path.join(directory, user, name_job)
-    import sys
 
-    print(full_path, file=sys.stderr)
+    logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job})
     # check if API_URL env variable is set
     file_data = {"name": name_job, "file": filename, "user": user}
     response = requests.get(
         urljoin(settings.API_URL, "/api/download"), params=file_data
     )
-    # check if file is in the response
-    print(response, file=sys.stderr)
     file = response.content
 
     if not os.path.exists(full_path):
@@ -130,18 +138,26 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
     )
 
     docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+    id = ObjectId()
 
-    call_openai_api(docs, full_path, self)
+    call_openai_api(docs, full_path, id, self)
     tokens = count_tokens_docs(docs)
     self.update_state(state="PROGRESS", meta={"current": 100})
 
     if sample:
         for i in range(min(5, len(raw_docs))):
-            print(raw_docs[i].text)
+            logging.info(f"Sample document {i}: {raw_docs[i]}")
 
     # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
     # and send them to the server (provide user and name in form)
-    file_data = {"name": name_job, "user": user, "tokens":tokens}
+    file_data = {
+        "name": name_job,
+        "user": user,
+        "tokens": tokens,
+        "retriever": retriever,
+        "id": str(id),
+        "type": "local",
+    }
     if settings.VECTOR_STORE == "faiss":
         files = {
             "file_faiss": open(full_path + "/index.faiss", "rb"),
@@ -150,9 +166,6 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         response = requests.post(
             urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
         )
-        response = requests.get(
-            urljoin(settings.API_URL, "/api/delete_old?path=" + full_path)
-        )
     else:
         response = requests.post(
             urljoin(settings.API_URL, "/api/upload_index"), data=file_data
@@ -171,7 +184,18 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
     }
 
 
-def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
+def remote_worker(
+    self,
+    source_data,
+    name_job,
+    user,
+    loader,
+    directory="temp",
+    retriever="classic",
+    sync_frequency="never",
+    operation_mode="upload",
+    doc_id=None,
+):
     token_check = True
     min_tokens = 150
     max_tokens = 1250
@@ -180,6 +204,10 @@ def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
     if not os.path.exists(full_path):
         os.makedirs(full_path)
     self.update_state(state="PROGRESS", meta={"current": 1})
+    logging.info(
+        f"Remote job: {full_path}",
+        extra={"user": user, "job": name_job, source_data: source_data},
+    )
 
     remote_loader = RemoteCreator.create_loader(loader)
     raw_docs = remote_loader.load_data(source_data)
@@ -191,22 +219,37 @@ def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
         token_check=token_check,
     )
     # docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
-    call_openai_api(docs, full_path, self)
     tokens = count_tokens_docs(docs)
+    if operation_mode == "upload":
+        id = ObjectId()
+        call_openai_api(docs, full_path, id, self)
+    elif operation_mode == "sync":
+        if not doc_id or not ObjectId.is_valid(doc_id):
+            raise ValueError("doc_id must be provided for sync operation.")
+        id = ObjectId(doc_id)
+        call_openai_api(docs, full_path, id, self)
     self.update_state(state="PROGRESS", meta={"current": 100})
 
     # Proceed with uploading and cleaning as in the original function
-    file_data = {"name": name_job, "user": user, "tokens":tokens}
+    file_data = {
+        "name": name_job,
+        "user": user,
+        "tokens": tokens,
+        "retriever": retriever,
+        "id": str(id),
+        "type": loader,
+        "remote_data": source_data,
+        "sync_frequency": sync_frequency,
+    }
     if settings.VECTOR_STORE == "faiss":
         files = {
             "file_faiss": open(full_path + "/index.faiss", "rb"),
             "file_pkl": open(full_path + "/index.pkl", "rb"),
         }
-        
+
         requests.post(
             urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
         )
-        requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path))
     else:
         requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
 
@@ -215,23 +258,55 @@ def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
     return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}
 
 
-def count_tokens_docs(docs):
-    # Here we convert the docs list to a string and calculate the number of tokens the string represents.
-    # docs_content = (" ".join(docs))
-    docs_content = ""
-    for doc in docs:
-        docs_content += doc.page_content
-
-    tokens, total_price = num_tokens_from_string(
-        string=docs_content, encoding_name="cl100k_base"
-    )
-    # Here we print the number of tokens and the approx user cost with some visually appealing formatting.
-    return tokens
+def sync(
+    self,
+    source_data,
+    name_job,
+    user,
+    loader,
+    sync_frequency,
+    retriever,
+    doc_id=None,
+    directory="temp",
+):
+    try:
+        remote_worker(
+            self,
+            source_data,
+            name_job,
+            user,
+            loader,
+            directory,
+            retriever,
+            sync_frequency,
+            "sync",
+            doc_id,
+        )
+    except Exception as e:
+        return {"status": "error", "error": str(e)}
+    return {"status": "success"}
 
 
-def num_tokens_from_string(string: str, encoding_name: str) -> int:
-    # Function to convert string to tokens and estimate user cost.
-    encoding = tiktoken.get_encoding(encoding_name)
-    num_tokens = len(encoding.encode(string))
-    total_price = (num_tokens / 1000) * 0.0004
-    return num_tokens, total_price
+def sync_worker(self, frequency):
+    sync_counts = Counter()
+    sources = sources_collection.find()
+    for doc in sources:
+        if doc.get("sync_frequency") == frequency:
+            name = doc.get("name")
+            user = doc.get("user")
+            source_type = doc.get("type")
+            source_data = doc.get("remote_data")
+            retriever = doc.get("retriever")
+            doc_id = str(doc.get("_id"))
+            resp = sync(
+                self, source_data, name, user, source_type, frequency, retriever, doc_id
+            )
+            sync_counts["total_sync_count"] += 1
+            sync_counts[
+                "sync_success" if resp["status"] == "success" else "sync_failure"
+            ] += 1
+
+    return {
+        key: sync_counts[key]
+        for key in ["total_sync_count", "sync_success", "sync_failure"]
+    }

docker-compose-azure.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docker-compose-dev.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
 
   redis:

docker-compose-local.yaml

@@ -1,8 +1,8 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend
+    volumes:
+    - ./frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING

docker-compose-mock.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docker-compose.yaml

@@ -1,8 +1,8 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend
+    volumes:
+      - ./frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING
@@ -32,7 +32,7 @@ services:
 
   worker:
     build: ./application
-    command: celery -A application.app.celery worker -l INFO
+    command: celery -A application.app.celery worker -l INFO -B
     environment:
       - API_KEY=$API_KEY
       - EMBEDDINGS_KEY=$API_KEY

docs/package.json

@@ -7,8 +7,8 @@
   "license": "MIT",
   "dependencies": {
     "@vercel/analytics": "^1.1.1",
-    "docsgpt": "^0.3.7",
-    "next": "^14.1.1",
+    "docsgpt": "^0.4.1",
+    "next": "^14.2.12",
     "nextra": "^2.13.2",
     "nextra-theme-docs": "^2.13.2",
     "react": "^18.2.0",

docs/pages/Deploying/Quickstart.md

@@ -67,46 +67,3 @@ To run the setup on Windows, you have two options: using the Windows Subsystem f
 
 These steps should help you set up and run the project on Windows using either WSL or Git Bash/Command Prompt. 
 **Important:** Ensure that Docker is installed and properly configured on your Windows system for these steps to work.
-
-
-For WINDOWS:
-
-To run the given setup on Windows, you can use the Windows Subsystem for Linux (WSL) or a Git Bash terminal to execute similar commands. Here are the steps adapted for Windows:
-
-Option 1: Using Windows Subsystem for Linux (WSL):
-
-1. Install WSL if you haven't already. You can follow the official Microsoft documentation for installation: (https://learn.microsoft.com/en-us/windows/wsl/install).
-2. After setting up WSL, open the WSL terminal.
-3. Clone the repository and create the `.env` file:
-   ```bash
-   git clone https://github.com/arc53/DocsGPT.git
-   cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
-   ```
-4. Run the following command to start the setup with Docker Compose:
-    ```bash
-    ./run-with-docker-compose.sh
-    ```
-5. Open your web browser and navigate to http://localhost:5173/.
-6. To stop the setup, just press **Ctrl + C** in the WSL terminal.
-
-Option 2: Using Git Bash or Command Prompt (CMD):
-
-1. Install Git for Windows if you haven't already. You can download it from the official website: (https://gitforwindows.org/).
-2. Open Git Bash or Command Prompt.
-3. Clone the repository and create the `.env` file:
-   ```bash
-   git clone https://github.com/arc53/DocsGPT.git
-   cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
-   ```
-4. Run the following command to start the setup with Docker Compose:
-   ```bash
-   ./run-with-docker-compose.sh
-   ```
-5. Open your web browser and navigate to http://localhost:5173/.
-6. To stop the setup, just press **Ctrl + C** in the Git Bash or Command Prompt terminal.
-
-These steps should help you set up and run the project on Windows using either WSL or Git Bash/Command Prompt. Make sure you have Docker installed and properly configured on your Windows system for this to work.

docs/pages/Extensions/react-widget.md

@@ -17,25 +17,31 @@ Now, you can use the widget in your component like this :
 ```jsx
 <DocsGPTWidget
   apiHost="https://your-docsgpt-api.com"
-  selectDocs="local/docs.zip"
   apiKey=""
-  avatar = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png",
-  title = "Get AI assistance",
-  description = "DocsGPT's AI Chatbot is here to help",
-  heroTitle = "Welcome to DocsGPT !",
+  avatar = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png"
+  title = "Get AI assistance"
+  description = "DocsGPT's AI Chatbot is here to help"
+  heroTitle = "Welcome to DocsGPT !"
   heroDescription="This chatbot is built with DocsGPT and utilises GenAI, 
   please review important information using sources."
+  theme = "dark"
+  buttonIcon = "https://your-icon"
+  buttonBg = "#222327"
 />
 ```
-DocsGPTWidget takes 8 **props** with default fallback values:
+To tailor the widget to your needs, you can configure the following props in your component:
 1. `apiHost` — The URL of your DocsGPT API.
-2. `selectDocs` — The documentation source that you want to use for your widget (e.g. `default` or `local/docs1.zip`).
+2. `theme` — Allows to select your specific theme (dark or light).
 3. `apiKey` — Usually, it's empty.
 4. `avatar`: Specifies the URL of the avatar or image representing the chatbot.
 5. `title`: Sets the title text displayed in the chatbot interface.
 6. `description`: Provides a brief description of the chatbot's purpose or functionality.
 7. `heroTitle`: Displays a welcome title when users interact with the chatbot.
 8. `heroDescription`: Provide additional introductory text or information about the chatbot's capabilities.
+9. `buttonIcon`: Specifies the url of the icon image for the widget.
+10. `buttonBg`: Allows to specify the Background color of the widget.
+11. `size`: Sets the size of the widget ( small, medium).
+
 
 ### How to use DocsGPTWidget with [Nextra](https://nextra.site/) (Next.js + MDX)
 Install your widget as described above and then go to your `pages/` folder and create a new file `_app.js` with the following content:
@@ -55,22 +61,30 @@ export default function MyApp({ Component, pageProps }) {
 ```html
 <!DOCTYPE html>
 <html lang="en">
-<head>
-  <meta charset="UTF-8" />
-  <meta http-equiv="X-UA-Compatible" content="IE=edge" />
-  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>DocsGPT Widget</title>
-</head>
-<body>
-  <div id="app"></div>
-  <!-- Include the widget script from dist/modern or dist/legacy -->
-  <script src="https://unpkg.com/docsgpt/dist/modern/main.js" type="module"></script>
-  <script type="module">
-    window.onload = function() {
-      renderDocsGPTWidget('app');
-    }
-  </script>
-</body>
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta http-equiv="X-UA-Compatible" content="ie=edge" />
+    <title>HTML + CSS</title>
+    <link rel="stylesheet" href="styles.css" />
+  </head>
+  <body>
+    <h1>This is a simple HTML + CSS template!</h1>
+    <div id="app"></div>
+    <!-- Include the widget script from dist/modern or dist/legacy -->
+    <script
+      src="https://unpkg.com/docsgpt/dist/modern/main.js"
+      type="module"
+    ></script>
+    <script type="module">
+      window.onload = function () {
+        renderDocsGPTWidget("app", {
+          apiKey: "",
+          size: "medium",
+        });
+      };
+    </script>
+  </body>
 </html>
 ```
 To link the widget to your api and your documents you can pass parameters to the renderDocsGPTWidget('div id', { parameters }).
@@ -82,22 +96,24 @@ To link the widget to your api and your documents you can pass parameters to the
   <meta http-equiv="X-UA-Compatible" content="IE=edge" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <title>DocsGPT Widget</title>
+  <script src="https://unpkg.com/docsgpt/dist/modern/main.js" type="module"></script>  
 </head>
 <body>
   <div id="app"></div>
   <!-- Include the widget script from dist/modern or dist/legacy -->
-  <script src="https://unpkg.com/docsgpt/dist/modern/main.js" type="module"></script>
   <script type="module">
     window.onload = function() {
       renderDocsGPTWidget('app', {
         apiHost: 'http://localhost:7001',
-        selectDocs: 'default',
-        apiKey: '',
+        apiKey:"",
         avatar: 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
         title: 'Get AI assistance',
         description: "DocsGPT's AI Chatbot is here to help",
         heroTitle: 'Welcome to DocsGPT!',
-        heroDescription: 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.'
+        heroDescription: 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.',
+        theme:"dark",
+        buttonIcon:"https://your-icon",
+        buttonBg:"#222327"
       });
     }
   </script>

docs/pages/Guides/How-to-use-different-LLM.mdx

@@ -36,6 +36,14 @@ List of latest supported LLMs are https://github.com/arc53/DocsGPT/blob/main/app
 Visit application/llm and select the file of your selected llm and there you will find the speicifc requirements needed to be filled in order to use it,i.e API key of that llm.
 </Steps>
 
+### For OpenAI-Compatible Endpoints:
+DocsGPT supports the use of OpenAI-compatible endpoints through base URL substitution. This feature allows you to use alternative AI models or services that implement the OpenAI API interface.
+
+
+Set the OPENAI_BASE_URL in your environment. You can change .env file with OPENAI_BASE_URL with the desired base URL or docker-compose.yml file and add the environment variable to the backend container.
+
+> Make sure you have the right API_KEY and correct LLM_NAME.
+
 
 
 

docs/pages/_app.mdx

@@ -4,7 +4,7 @@ export default function MyApp({ Component, pageProps }) {
   return (
     <>
       <Component {...pageProps} />
-        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" />
+        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" theme="dark" />
     </>
   )
 }

extensions/chrome/package-lock.json

@@ -107,12 +107,12 @@
       }
     },
     "node_modules/braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dev": true,
       "dependencies": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       },
       "engines": {
         "node": ">=8"
@@ -260,9 +260,9 @@
       }
     },
     "node_modules/fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "dev": true,
       "dependencies": {
         "to-regex-range": "^5.0.1"
@@ -884,12 +884,12 @@
       "dev": true
     },
     "braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dev": true,
       "requires": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       }
     },
     "camelcase-css": {
@@ -1000,9 +1000,9 @@
       }
     },
     "fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "dev": true,
       "requires": {
         "to-regex-range": "^5.0.1"

extensions/react-widget/README.md

@@ -27,15 +27,18 @@ To link the widget to your api and your documents you can pass parameters to the
 
     const App = () => {
       return <DocsGPTWidget
-             apiHost = 'http://localhost:7001',
-             selectDocs = 'default',
-             apiKey = '',
-             avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
-             title = 'Get AI assistance',
-             description = 'DocsGPT\'s AI Chatbot is here to help',
-             heroTitle = 'Welcome to DocsGPT !',
-             heroDescription='This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.'
-             />;
+               apiHost="https://your-docsgpt-api.com"
+               apiKey=""
+               avatar = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png"
+               title = "Get AI assistance"
+               description = "DocsGPT's AI Chatbot is here to help"
+               heroTitle = "Welcome to DocsGPT !"
+               heroDescription="This chatbot is built with DocsGPT and utilises GenAI, 
+               please review important information using sources."
+               theme = "dark"
+               buttonIcon = "https://your-icon"
+               buttonBg = "#222327"
+          />;
     };
 ```
 
@@ -80,15 +83,17 @@ To link the widget to your api and your documents you can pass parameters to the
         <script src="https://unpkg.com/docsgpt/dist/modern/main.js" type="module"></script>
         <script type="module">
           window.onload = function() {
-            renderDocsGPTWidget('app', , {
+            renderDocsGPTWidget('app', {
               apiHost: 'http://localhost:7001',
-              selectDocs: 'default',
-              apiKey: '',
+              apiKey:"",
               avatar: 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
               title: 'Get AI assistance',
               description: "DocsGPT's AI Chatbot is here to help",
-              heroTitle: 'Welcome to DocsGPT !',
-              heroDescription: 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.'
+              heroTitle: 'Welcome to DocsGPT!',
+              heroDescription: 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.',
+              theme:"dark",
+              buttonIcon:"https://your-icon.svg",
+              buttonBg:"#222327"
             });
           }
         </script>

extensions/react-widget/package-lock.json

@@ -1,16 +1,15 @@
 {
   "name": "docsgpt",
-  "version": "0.3.7",
+  "version": "0.4.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "docsgpt",
-      "version": "0.3.7",
+      "version": "0.4.2",
       "license": "Apache-2.0",
       "dependencies": {
         "@babel/plugin-transform-flow-strip-types": "^7.23.3",
-        "@bpmn-io/snarkdown": "^2.2.0",
         "@parcel/resolver-glob": "^2.12.0",
         "@parcel/transformer-svg-react": "^2.12.0",
         "@parcel/transformer-typescript-tsc": "^2.12.0",
@@ -22,6 +21,7 @@
         "flow-bin": "^0.229.2",
         "i": "^0.3.7",
         "install": "^0.13.0",
+        "markdown-it": "^14.1.0",
         "npm": "^10.5.0",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
@@ -34,6 +34,7 @@
         "@parcel/packager-ts": "^2.12.0",
         "@parcel/transformer-typescript-types": "^2.12.0",
         "@types/dompurify": "^3.0.5",
+        "@types/markdown-it": "^14.1.2",
         "@types/react": "^18.3.3",
         "@types/react-dom": "^18.3.0",
         "babel-loader": "^8.0.4",
@@ -1837,11 +1838,6 @@
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@bpmn-io/snarkdown": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/@bpmn-io/snarkdown/-/snarkdown-2.2.0.tgz",
-      "integrity": "sha512-bVD7FIoaBDZeCJkMRgnBPDeptPlto87wt2qaCjf5t8iLaevDmTPaREd6FpBEGsHlUdHFFZWRk4qAoEC5So2M0Q=="
-    },
     "node_modules/@emotion/is-prop-valid": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.2.2.tgz",
@@ -4602,32 +4598,10 @@
         "@types/trusted-types": "*"
       }
     },
-    "node_modules/@types/eslint": {
-      "version": "8.56.10",
-      "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.10.tgz",
-      "integrity": "sha512-Shavhk87gCtY2fhXDctcfS3e6FdxWkCx1iUZ9eEUbh7rTqlZT0/IzOkCOVt0fCjcFuZ9FPYfuezTBImfHCDBGQ==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@types/estree": "*",
-        "@types/json-schema": "*"
-      }
-    },
-    "node_modules/@types/eslint-scope": {
-      "version": "3.7.7",
-      "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz",
-      "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@types/eslint": "*",
-        "@types/estree": "*"
-      }
-    },
     "node_modules/@types/estree": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz",
-      "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==",
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
+      "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==",
       "dev": true,
       "peer": true
     },
@@ -4637,14 +4611,39 @@
       "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
       "dev": true
     },
+    "node_modules/@types/linkify-it": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-5.0.0.tgz",
+      "integrity": "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/@types/markdown-it": {
+      "version": "14.1.2",
+      "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-14.1.2.tgz",
+      "integrity": "sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@types/linkify-it": "^5",
+        "@types/mdurl": "^2"
+      }
+    },
+    "node_modules/@types/mdurl": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-2.0.0.tgz",
+      "integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/node": {
-      "version": "20.13.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.13.0.tgz",
-      "integrity": "sha512-FM6AOb3khNkNIXPnHFDYaHerSv8uN22C91z098AnGccVu+Pcdhi+pNUFDi0iLmPIsVE0JBD0KVS7mzUYt4nRzQ==",
+      "version": "22.5.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.5.tgz",
+      "integrity": "sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "undici-types": "~5.26.4"
+        "undici-types": "~6.19.2"
       }
     },
     "node_modules/@types/parse-json": {
@@ -4869,9 +4868,9 @@
       "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ=="
     },
     "node_modules/acorn": {
-      "version": "8.11.3",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
-      "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==",
+      "version": "8.12.1",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
+      "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==",
       "dev": true,
       "peer": true,
       "bin": {
@@ -4881,10 +4880,10 @@
         "node": ">=0.4.0"
       }
     },
-    "node_modules/acorn-import-assertions": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz",
-      "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==",
+    "node_modules/acorn-import-attributes": {
+      "version": "1.9.5",
+      "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz",
+      "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==",
       "dev": true,
       "peer": true,
       "peerDependencies": {
@@ -4930,8 +4929,7 @@
     "node_modules/argparse": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
-      "dev": true
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
     },
     "node_modules/babel-loader": {
       "version": "8.3.0",
@@ -5232,73 +5230,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/css-select": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
-      "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "boolbase": "^1.0.0",
-        "css-what": "^6.1.0",
-        "domhandler": "^5.0.2",
-        "domutils": "^3.0.1",
-        "nth-check": "^2.0.1"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/fb55"
-      }
-    },
-    "node_modules/css-select/node_modules/dom-serializer": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
-      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "domelementtype": "^2.3.0",
-        "domhandler": "^5.0.2",
-        "entities": "^4.2.0"
-      },
-      "funding": {
-        "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
-      }
-    },
-    "node_modules/css-select/node_modules/domhandler": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
-      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "domelementtype": "^2.3.0"
-      },
-      "engines": {
-        "node": ">= 4"
-      },
-      "funding": {
-        "url": "https://github.com/fb55/domhandler?sponsor=1"
-      }
-    },
-    "node_modules/css-select/node_modules/domutils": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
-      "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "dom-serializer": "^2.0.0",
-        "domelementtype": "^2.3.0",
-        "domhandler": "^5.0.3"
-      },
-      "funding": {
-        "url": "https://github.com/fb55/domutils?sponsor=1"
-      }
-    },
     "node_modules/css-to-react-native": {
       "version": "3.2.0",
       "resolved": "https://registry.npmjs.org/css-to-react-native/-/css-to-react-native-3.2.0.tgz",
@@ -5309,21 +5240,6 @@
         "postcss-value-parser": "^4.0.2"
       }
     },
-    "node_modules/css-tree": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz",
-      "integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "mdn-data": "2.0.30",
-        "source-map-js": "^1.0.1"
-      },
-      "engines": {
-        "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0"
-      }
-    },
     "node_modules/css-what": {
       "version": "6.1.0",
       "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
@@ -5335,45 +5251,6 @@
         "url": "https://github.com/sponsors/fb55"
       }
     },
-    "node_modules/csso": {
-      "version": "5.0.5",
-      "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz",
-      "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "css-tree": "~2.2.0"
-      },
-      "engines": {
-        "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0",
-        "npm": ">=7.0.0"
-      }
-    },
-    "node_modules/csso/node_modules/css-tree": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz",
-      "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "mdn-data": "2.0.28",
-        "source-map-js": "^1.0.1"
-      },
-      "engines": {
-        "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0",
-        "npm": ">=7.0.0"
-      }
-    },
-    "node_modules/csso/node_modules/mdn-data": {
-      "version": "2.0.28",
-      "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz",
-      "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==",
-      "dev": true,
-      "optional": true,
-      "peer": true
-    },
     "node_modules/csstype": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
@@ -5506,9 +5383,9 @@
       }
     },
     "node_modules/enhanced-resolve": {
-      "version": "5.16.1",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.16.1.tgz",
-      "integrity": "sha512-4U5pNsuDl0EhuZpq46M5xPslstkviJuhrdobaRDBk2Jy2KO37FDAJl4lb2KlNabxT0m4MTK2UHNrsAcphE8nyw==",
+      "version": "5.17.1",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz",
+      "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==",
       "dev": true,
       "peer": true,
       "dependencies": {
@@ -5548,9 +5425,9 @@
       }
     },
     "node_modules/es-module-lexer": {
-      "version": "1.5.3",
-      "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.3.tgz",
-      "integrity": "sha512-i1gCgmR9dCl6Vil6UKPI/trA69s08g/syhiDK9TG0Nf1RJjjFI+AzoWW7sPufzkgYAn861skuCwJa0pIIHYxvg==",
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
+      "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==",
       "dev": true,
       "peer": true
     },
@@ -6256,6 +6133,15 @@
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
       "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
     },
+    "node_modules/linkify-it": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz",
+      "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==",
+      "license": "MIT",
+      "dependencies": {
+        "uc.micro": "^2.0.0"
+      }
+    },
     "node_modules/lmdb": {
       "version": "2.8.5",
       "resolved": "https://registry.npmjs.org/lmdb/-/lmdb-2.8.5.tgz",
@@ -6361,13 +6247,28 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/mdn-data": {
-      "version": "2.0.30",
-      "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz",
-      "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==",
-      "dev": true,
-      "optional": true,
-      "peer": true
+    "node_modules/markdown-it": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz",
+      "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==",
+      "license": "MIT",
+      "dependencies": {
+        "argparse": "^2.0.1",
+        "entities": "^4.4.0",
+        "linkify-it": "^5.0.0",
+        "mdurl": "^2.0.0",
+        "punycode.js": "^2.3.1",
+        "uc.micro": "^2.1.0"
+      },
+      "bin": {
+        "markdown-it": "bin/markdown-it.mjs"
+      }
+    },
+    "node_modules/mdurl": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
+      "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==",
+      "license": "MIT"
     },
     "node_modules/merge-stream": {
       "version": "2.0.0",
@@ -6377,9 +6278,9 @@
       "peer": true
     },
     "node_modules/micromatch": {
-      "version": "4.0.7",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
-      "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
+      "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
       "dependencies": {
         "braces": "^3.0.3",
         "picomatch": "^2.3.1"
@@ -9245,6 +9146,15 @@
         "node": ">=6"
       }
     },
+    "node_modules/punycode.js": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",
+      "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/randombytes": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
@@ -9484,20 +9394,6 @@
         "source-map": "^0.6.0"
       }
     },
-    "node_modules/srcset": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/srcset/-/srcset-5.0.1.tgz",
-      "integrity": "sha512-/P1UYbGfJVlxZag7aABNRrulEXAwCSDo7fklafOQrantuPTDmYgijJMks2zusPCVzgW9+4P69mq7w6pYuZpgxw==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "engines": {
-        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/stable": {
       "version": "0.1.8",
       "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz",
@@ -9564,33 +9460,6 @@
       "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz",
       "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ=="
     },
-    "node_modules/svgo": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/svgo/-/svgo-3.3.2.tgz",
-      "integrity": "sha512-OoohrmuUlBs8B8o6MB2Aevn+pRIH9zDALSR+6hhqVfa6fRwG/Qw9VUMSMW9VNg2CFc/MTIfabtdOVl9ODIJjpw==",
-      "dev": true,
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "@trysound/sax": "0.2.0",
-        "commander": "^7.2.0",
-        "css-select": "^5.1.0",
-        "css-tree": "^2.3.1",
-        "css-what": "^6.1.0",
-        "csso": "^5.0.5",
-        "picocolors": "^1.0.0"
-      },
-      "bin": {
-        "svgo": "bin/svgo"
-      },
-      "engines": {
-        "node": ">=14.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/svgo"
-      }
-    },
     "node_modules/tapable": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
@@ -9614,9 +9483,9 @@
       }
     },
     "node_modules/terser": {
-      "version": "5.31.0",
-      "resolved": "https://registry.npmjs.org/terser/-/terser-5.31.0.tgz",
-      "integrity": "sha512-Q1JFAoUKE5IMfI4Z/lkE/E6+SwgzO+x4tq4v1AyBLRj8VSYvRO6A/rQrPg1yud4g0En9EKI1TvFRF2tQFcoUkg==",
+      "version": "5.33.0",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.33.0.tgz",
+      "integrity": "sha512-JuPVaB7s1gdFKPKTelwUyRq5Sid2A3Gko2S0PncwdBq7kN9Ti9HPWDQ06MPsEDGsZeVESjKEnyGy68quBk1w6g==",
       "dev": true,
       "peer": true,
       "dependencies": {
@@ -9747,10 +9616,16 @@
         "node": ">=14.17"
       }
     },
+    "node_modules/uc.micro": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz",
+      "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
+      "license": "MIT"
+    },
     "node_modules/undici-types": {
-      "version": "5.26.5",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
-      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
       "dev": true,
       "peer": true
     },
@@ -9841,9 +9716,9 @@
       }
     },
     "node_modules/watchpack": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.1.tgz",
-      "integrity": "sha512-8wrBCMtVhqcXP2Sup1ctSkga6uc2Bx0IIvKyT7yTFier5AXHooSI+QyQQAtTb7+E0IUCCKyTFmXqdqgum2XWGg==",
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz",
+      "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==",
       "dev": true,
       "peer": true,
       "dependencies": {
@@ -9860,22 +9735,21 @@
       "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw=="
     },
     "node_modules/webpack": {
-      "version": "5.91.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.91.0.tgz",
-      "integrity": "sha512-rzVwlLeBWHJbmgTC/8TvAcu5vpJNII+MelQpylD4jNERPwpBJOE2lEcko1zJX3QJeLjTTAnQxn/OJ8bjDzVQaw==",
+      "version": "5.94.0",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.94.0.tgz",
+      "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@types/eslint-scope": "^3.7.3",
         "@types/estree": "^1.0.5",
         "@webassemblyjs/ast": "^1.12.1",
         "@webassemblyjs/wasm-edit": "^1.12.1",
         "@webassemblyjs/wasm-parser": "^1.12.1",
         "acorn": "^8.7.1",
-        "acorn-import-assertions": "^1.9.0",
+        "acorn-import-attributes": "^1.9.5",
         "browserslist": "^4.21.10",
         "chrome-trace-event": "^1.0.2",
-        "enhanced-resolve": "^5.16.0",
+        "enhanced-resolve": "^5.17.1",
         "es-module-lexer": "^1.2.1",
         "eslint-scope": "5.1.1",
         "events": "^3.2.0",

extensions/react-widget/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docsgpt",
-  "version": "0.3.9",
+  "version": "0.4.2",
   "private": false,
   "description": "DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.",
   "source": "./src/index.html",
@@ -31,6 +31,7 @@
   },
   "scripts": {
     "build": "parcel build src/main.tsx --public-url ./",
+    "build:react": "parcel build src/index.ts",
     "dev": "parcel src/index.html -p 3000",
     "test": "jest",
     "lint": "eslint",
@@ -39,7 +40,6 @@
   },
   "dependencies": {
     "@babel/plugin-transform-flow-strip-types": "^7.23.3",
-    "@bpmn-io/snarkdown": "^2.2.0",
     "@parcel/resolver-glob": "^2.12.0",
     "@parcel/transformer-svg-react": "^2.12.0",
     "@parcel/transformer-typescript-tsc": "^2.12.0",
@@ -51,6 +51,7 @@
     "flow-bin": "^0.229.2",
     "i": "^0.3.7",
     "install": "^0.13.0",
+    "markdown-it": "^14.1.0",
     "npm": "^10.5.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
@@ -63,6 +64,7 @@
     "@parcel/packager-ts": "^2.12.0",
     "@parcel/transformer-typescript-types": "^2.12.0",
     "@types/dompurify": "^3.0.5",
+    "@types/markdown-it": "^14.1.2",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "babel-loader": "^8.0.4",

extensions/react-widget/publish.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+## chmod +x publish.sh - to upgrade ownership
+set -e
+cat package.json >> package_copy.json
+cat package-lock.json >> package-lock_copy.json
+publish_package() {
+  PACKAGE_NAME=$1
+  BUILD_COMMAND=$2
+  # Update package name in package.json
+  jq --arg name "$PACKAGE_NAME" '.name=$name' package.json > temp.json && mv temp.json package.json
+
+  # Remove 'target' key if the package name is 'docsgpt-react'
+  if [ "$PACKAGE_NAME" = "docsgpt-react" ]; then
+    jq 'del(.targets)' package.json > temp.json && mv temp.json package.json
+  fi
+
+  if [ -d "dist" ]; then
+    echo "Deleting existing dist directory..."
+    rm -rf dist
+  fi
+
+  npm version patch
+
+  npm run "$BUILD_COMMAND"
+
+  # Publish to npm
+  npm publish
+  # Clean up
+  mv package_copy.json package.json
+  mv package-lock_copy.json package-lock.json
+  echo "Published ${PACKAGE_NAME}"
+}
+
+# Publish docsgpt package
+publish_package "docsgpt" "build"
+
+# Publish docsgpt-react package
+publish_package "docsgpt-react" "build:react"
+
+
+rm -rf package_copy.json
+rm -rf package-lock_copy.json
+echo "---Process completed---"

extensions/react-widget/src/assets/dislike.svg

@@ -0,0 +1,4 @@
+<svg width="14" height="14" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.37776 10.1001V12.9C6.37776 13.457 6.599 13.9911 6.99282 14.3849C7.38664 14.7788 7.92077 15 8.47772 15L11.2777 8.70011V1.00025H3.38181C3.04419 0.996436 2.71656 1.11477 2.45929 1.33344C2.20203 1.55212 2.03246 1.8564 1.98184 2.19023L1.01585 8.49012C0.985398 8.69076 0.998931 8.89563 1.05551 9.09053C1.1121 9.28543 1.21038 9.46569 1.34355 9.61884C1.47671 9.77198 1.64159 9.89434 1.82674 9.97744C2.01189 10.0605 2.2129 10.1024 2.41583 10.1001H6.37776ZM11.2777 1.00025H13.1466C13.5428 0.993247 13.9277 1.13195 14.2284 1.39002C14.5291 1.64809 14.7245 2.00758 14.7776 2.40023V7.30014C14.7245 7.69279 14.5291 8.05227 14.2284 8.31035C13.9277 8.56842 13.5428 8.70712 13.1466 8.70011H11.2777" fill="none"/>
+<path d="M11.2777 8.70011L8.47772 15C7.92077 15 7.38664 14.7788 6.99282 14.3849C6.599 13.9911 6.37776 13.457 6.37776 12.9V10.1001H2.41583C2.2129 10.1024 2.01189 10.0605 1.82674 9.97744C1.64159 9.89434 1.47671 9.77198 1.34355 9.61884C1.21038 9.46569 1.1121 9.28543 1.05551 9.09053C0.998931 8.89563 0.985398 8.69076 1.01585 8.49012L1.98184 2.19023C2.03246 1.8564 2.20203 1.55212 2.45929 1.33344C2.71656 1.11477 3.04419 0.996436 3.38181 1.00025H11.2777M11.2777 8.70011V1.00025M11.2777 8.70011H13.1466C13.5428 8.70712 13.9277 8.56842 14.2284 8.31035C14.5291 8.05227 14.7245 7.69279 14.7776 7.30014V2.40023C14.7245 2.00758 14.5291 1.64809 14.2284 1.39002C13.9277 1.13195 13.5428 0.993247 13.1466 1.00025H11.2777" stroke="current" stroke-width="1.4" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

extensions/react-widget/src/assets/like.svg

@@ -0,0 +1,4 @@
+<svg width="14" height="14" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
+<path d="M9.39995 5.89997V3.09999C9.39995 2.54304 9.1787 2.0089 8.78487 1.61507C8.39105 1.22125 7.85691 1 7.29996 1L4.49998 7.29996V14.9999H12.3959C12.7336 15.0037 13.0612 14.8854 13.3185 14.6667C13.5757 14.448 13.7453 14.1437 13.7959 13.8099L14.7619 7.50996C14.7924 7.30931 14.7788 7.10444 14.7222 6.90954C14.6657 6.71464 14.5674 6.53437 14.4342 6.38123C14.301 6.22808 14.1362 6.10572 13.951 6.02262C13.7659 5.93952 13.5649 5.89767 13.3619 5.89997H9.39995ZM4.49998 14.9999H2.39999C2.02869 14.9999 1.6726 14.8524 1.41005 14.5899C1.1475 14.3273 1 13.9712 1 13.5999V8.69995C1 8.32865 1.1475 7.97256 1.41005 7.71001C1.6726 7.44746 2.02869 7.29996 2.39999 7.29996H4.49998" fill="none"/>
+<path d="M4.49998 7.29996L7.29996 1C7.85691 1 8.39105 1.22125 8.78487 1.61507C9.1787 2.0089 9.39995 2.54304 9.39995 3.09999V5.89997H13.3619C13.5649 5.89767 13.7659 5.93952 13.951 6.02262C14.1362 6.10572 14.301 6.22808 14.4342 6.38123C14.5674 6.53437 14.6657 6.71464 14.7223 6.90954C14.7788 7.10444 14.7924 7.30931 14.7619 7.50996L13.7959 13.8099C13.7453 14.1437 13.5757 14.448 13.3185 14.6667C13.0612 14.8854 12.7336 15.0037 12.3959 14.9999H4.49998M4.49998 7.29996V14.9999M4.49998 7.29996H2.39999C2.02869 7.29996 1.6726 7.44746 1.41005 7.71001C1.1475 7.97256 1 8.32865 1 8.69995V13.5999C1 13.9712 1.1475 14.3273 1.41005 14.5899C1.6726 14.8524 2.02869 14.9999 2.39999 14.9999H4.49998" stroke="current" stroke-width="1.39999" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

extensions/react-widget/src/assets/message.svg

@@ -1,7 +0,0 @@
-<svg width="36" height="36" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M4.37891 9.44824H7.75821" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M11.1377 9.44824H12.8273" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M4.37891 6.06934H6.06856" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M9.44824 6.06934H12.8276" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M16.2069 11.1379C16.2069 11.5861 16.0289 12.0158 15.712 12.3327C15.3951 12.6496 14.9654 12.8276 14.5172 12.8276H4.37931L1 16.2069V2.68965C1 2.24153 1.17802 1.81176 1.49489 1.49489C1.81176 1.17802 2.24153 1 2.68965 1H14.5172C14.9654 1 15.3951 1.17802 15.712 1.49489C16.0289 1.81176 16.2069 2.24153 16.2069 2.68965V11.1379Z" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-</svg>

extensions/react-widget/src/components/DocsGPTWidget.tsx

@@ -1,13 +1,40 @@
 "use client";
-import React from 'react'
+import React, { useRef } from 'react'
 import DOMPurify from 'dompurify';
-import snarkdown from '@bpmn-io/snarkdown';
 import styled, { keyframes, createGlobalStyle } from 'styled-components';
 import { PaperPlaneIcon, RocketIcon, ExclamationTriangleIcon, Cross2Icon } from '@radix-ui/react-icons';
-import MessageIcon from '../assets/message.svg';
-import { MESSAGE_TYPE, Query, Status } from '../types/index';
-import { fetchAnswerStreaming } from '../requests/streamingApi';
-
+import { FEEDBACK, MESSAGE_TYPE, Query, Status, WidgetProps } from '../types/index';
+import { fetchAnswerStreaming, sendFeedback } from '../requests/streamingApi';
+import { ThemeProvider } from 'styled-components';
+import Like from "../assets/like.svg"
+import Dislike from "../assets/dislike.svg"
+import MarkdownIt from 'markdown-it';
+const themes = {
+  dark: {
+    bg: '#222327',
+    text: '#fff',
+    primary: {
+      text: "#FAFAFA",
+      bg: '#222327'
+    },
+    secondary: {
+      text: "#A1A1AA",
+      bg: "#38383b"
+    }
+  },
+  light: {
+    bg: '#fff',
+    text: '#000',
+    primary: {
+      text: "#222327",
+      bg: "#fff"
+    },
+    secondary: {
+      text: "#A1A1AA",
+      bg: "#F6F6F6"
+    }
+  }
+}
 const GlobalStyles = createGlobalStyle`
 .response pre {
     padding: 8px;
@@ -16,6 +43,7 @@ const GlobalStyles = createGlobalStyle`
     border-radius: 6px;
     overflow-x: auto;
     background-color: #1B1C1F;
+    color: #fff !important;
 }
 .response h1{
   font-size: 20px;
@@ -26,40 +54,64 @@ const GlobalStyles = createGlobalStyle`
 .response h3{
   font-size: 16px;
 }
+.response p{
+  margin:0px;
+}
 .response code:not(pre code){
   border-radius: 6px;
   padding: 1px 3px 1px 3px;
   font-size: 12px;
   display: inline-block;
   background-color: #646464;
+  color: #fff !important;
+}
+.response code {
+  white-space: pre-wrap !important;
+  line-break: loose !important;
 }
 `;
-const WidgetContainer = styled.div`
+const Overlay = styled.div`
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background-color: rgba(0, 0, 0, 0.5);
+  z-index: 999;
+  transition: opacity 0.5s;
+`
+const WidgetContainer = styled.div<{ modal: boolean }>`
     display: block;
     position: fixed;
-    right: 10px;
-    bottom: 10px;
+    right: ${props => props.modal ? '50%' : '10px'};
+    bottom: ${props => props.modal ? '50%' : '10px'};
     z-index: 1000;
     display: flex;
+    ${props => props.modal &&
+    "transform : translate(50%,50%);"
+  }
     flex-direction: column;
     align-items: center;
     text-align: left;
+    @media only screen and (max-width: 768px) {
+    max-height: 100vh !important;
+    overflow: auto;
+    }
 `;
 const StyledContainer = styled.div`
-    display: block;
+    display: flex;
     position: relative;
+    flex-direction: column;
+    justify-content: center;
     bottom: 0;
     left: 0;
-    width: 352px;
-    height: 407px;
-    max-height: 407px;
     border-radius: 0.75rem;
-    background-color: #222327;
+    background-color: ${props => props.theme.primary.bg};
     font-family: sans-serif;
     box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05), 0 2px 4px rgba(0, 0, 0, 0.1);
     transition: visibility 0.3s, opacity 0.3s;
 `;
-const FloatingButton = styled.div`
+const FloatingButton = styled.div<{ bgcolor: string }>`
     position: fixed;
     display: flex;
     z-index: 500;
@@ -70,7 +122,7 @@ const FloatingButton = styled.div`
     width: 5rem;
     height: 5rem;
     border-radius: 9999px;
-    background-image: linear-gradient(to bottom right, #5AF0EC, #E80D9D);
+    background: ${props => props.bgcolor};
     box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
     cursor: pointer;
     &:hover {
@@ -120,39 +172,62 @@ const ContentWrapper = styled.div`
 const Title = styled.h3`
     font-size: 1rem;
     font-weight: normal;
-    color: #FAFAFA;
+    color: ${props => props.theme.primary.text};
     margin-top: 0;
     margin-bottom: 0.25rem;
 `;
 
 const Description = styled.p`
     font-size: 0.85rem;
-    color: #A1A1AA;
+    color: ${props => props.theme.secondary.text};
     margin-top: 0;
 `;
-const Conversation = styled.div`
-    height: 16rem;
+
+const Conversation = styled.div<{ size: string }>`
+    min-height: 250px;
+    max-width: 968px;
+    height: ${props => props.size === 'large' ? '75vh' : props.size === 'medium' ? '70vh' : '320px'};
+    width: ${props => props.size === 'large' ? '60vw' : props.size === 'medium' ? '28vw' : '400px'};
     padding-inline: 0.5rem;
     border-radius: 0.375rem;
     text-align: left;
     overflow-y: auto;
     scrollbar-width: thin;
     scrollbar-color: #4a4a4a transparent; /* thumb color track color */
+    @media only screen and (max-width: 768px) {
+    width: 90vw !important;
+    }
+    @media only screen and (min-width:768px ) and (max-width: 1280px) {
+    width:${props => props.size === 'large' ? '90vw' : props.size === 'medium' ? '60vw' : '400px'} !important;
+    }
+`;
+const Feedback = styled.div`
+  background-color: transparent;
+  font-weight: normal;
+  gap: 12px;
+  display: flex;
+  padding: 6px;
+  clear: both;
 `;
-
 const MessageBubble = styled.div<{ type: MESSAGE_TYPE }>`
-    display: flex;
+    display: block;
     font-size: 16px;
-    justify-content: ${props => props.type === 'QUESTION' ? 'flex-end' : 'flex-start'};
-    margin: 0.5rem;
+    position: relative;
+    width: 100%;;
+    float: right;
+    margin: 0rem;
+    &:hover ${Feedback} * {
+    visibility: visible !important;
+  }
 `;
-const Message = styled.p<{ type: MESSAGE_TYPE }>`
+const Message = styled.div<{ type: MESSAGE_TYPE }>`
     background: ${props => props.type === 'QUESTION' ?
     'linear-gradient(to bottom right, #8860DB, #6D42C5)' :
-    '#38383b'};
-    color: #ffff;
+    props.theme.secondary.bg};
+    color: ${props => props.type === 'ANSWER' ? props.theme.primary.text : '#fff'};
     border: none;
-    max-width: 80%;
+    float: ${props => props.type === 'QUESTION' ? 'right' : 'left'};
+    max-width: ${props => props.type === 'ANSWER' ? '100%' : '80'};
     overflow: auto;
     margin: 4px;
     display: block;
@@ -190,35 +265,31 @@ const DotAnimation = styled.div`
 const Delay = styled(DotAnimation) <{ delay: number }>`
   animation-delay: ${props => props.delay + 'ms'};
 `;
-const PromptContainer = styled.form`
+const PromptContainer = styled.form<{ size: string }>`
   background-color: transparent;
-  height: 36px;
-  position: absolute;
-  bottom: 25px;
-  left: 24px;
-  right: 24px;
+  height: ${props => props.size == 'large' ? '60px' : '40px'};
+  margin: 16px;
   display: flex;
   justify-content: space-evenly;
 `;
 const StyledInput = styled.input`
-  width: 260px;
-  height: 36px;
+  width: 100%;
   border: 1px solid #686877;
   padding-left: 12px;
   background-color: transparent;
   font-size: 16px;
   border-radius: 6px;
-  color: #ffff;
+  color: ${props => props.theme.text};
   outline: none;
 `;
-const StyledButton = styled.button`
+const StyledButton = styled.button<{ size: string }>`
   display: flex;
   justify-content: center;
   align-items: center;
   background-image: linear-gradient(to bottom right, #5AF0EC, #E80D9D);
   border-radius: 6px;
-  width: 36px;
-  height: 36px;
+  min-width: ${props => props.size === 'large' ? '60px' : '36px'};
+  height: ${props => props.size === 'large' ? '60px' : '36px'};
   margin-left:8px;
   padding: 0px;
   border: none;
@@ -239,13 +310,14 @@ const HeroContainer = styled.div`
   align-items: middle;
   transform: translate(-50%, -50%);
   width: 80%;
+  max-width: 500px;
   background-image: linear-gradient(to bottom right, #5AF0EC, #ff1bf4);
   border-radius: 10px;
   margin: 0 auto;
   padding: 2px;
 `;
 const HeroWrapper = styled.div`
-  background-color: #222327;
+  background-color: ${props => props.theme.primary.bg};
   border-radius: 10px; 
   font-weight: normal;
   padding: 6px;
@@ -253,23 +325,23 @@ const HeroWrapper = styled.div`
   justify-content: space-between;
 `
 const HeroTitle = styled.h3`
-  color: #fff;
-  font-size: 17px;
+  color: ${props => props.theme.text};
   margin-bottom: 5px;
   padding: 2px;
 `;
 const HeroDescription = styled.p`
-  color: #fff;
+  color: ${props => props.theme.text};
   font-size: 14px;
   line-height: 1.5;
 `;
-const Hero = ({ title, description }: { title: string, description: string }) => {
+
+const Hero = ({ title, description, theme }: { title: string, description: string, theme: string }) => {
   return (
     <>
       <HeroContainer>
         <HeroWrapper>
-          <IconWrapper style={{ marginTop: '8px' }}>
-            <RocketIcon color='white' width={20} height={20} />
+          <IconWrapper style={{ marginTop: '12px' }}>
+            <RocketIcon color={theme === 'light' ? 'black' : 'white'} width={20} height={20} />
           </IconWrapper>
           <div>
             <HeroTitle>{title}</HeroTitle>
@@ -284,22 +356,28 @@ const Hero = ({ title, description }: { title: string, description: string }) =>
 };
 export const DocsGPTWidget = ({
   apiHost = 'https://gptcloud.arc53.com',
-  selectDocs = 'default',
   apiKey = '82962c9a-aa77-4152-94e5-a4f84fd44c6a',
   avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
   title = 'Get AI assistance',
   description = 'DocsGPT\'s AI Chatbot is here to help',
   heroTitle = 'Welcome to DocsGPT !',
-  heroDescription = 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.'
-}) => {
-
+  heroDescription = 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.',
+  size = 'small',
+  theme = 'dark',
+  buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/message.svg',
+  buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)',
+  collectFeedback = true
+}: WidgetProps) => {
   const [prompt, setPrompt] = React.useState('');
   const [status, setStatus] = React.useState<Status>('idle');
   const [queries, setQueries] = React.useState<Query[]>([])
   const [conversationId, setConversationId] = React.useState<string | null>(null)
   const [open, setOpen] = React.useState<boolean>(false)
   const [eventInterrupt, setEventInterrupt] = React.useState<boolean>(false); //click or scroll by user while autoScrolling
+  const isBubbleHovered = useRef<boolean>(false)
   const endMessageRef = React.useRef<HTMLDivElement | null>(null);
+  const md = new MarkdownIt();
+
   const handleUserInterrupt = () => {
     (status === 'loading') && setEventInterrupt(true);
   }
@@ -316,11 +394,40 @@ export const DocsGPTWidget = ({
     const lastChild = element?.children?.[element.children.length - 1]
     lastChild && scrollToBottom(lastChild)
   };
-
   React.useEffect(() => {
     !eventInterrupt && scrollToBottom(endMessageRef.current);
   }, [queries.length, queries[queries.length - 1]?.response]);
 
+  async function handleFeedback(feedback: FEEDBACK, index: number) {
+    let query = queries[index]
+    if (!query.response)
+      return;
+    if (query.feedback != feedback) {
+      sendFeedback({
+        question: query.prompt,
+        answer: query.response,
+        feedback: feedback,
+        apikey: apiKey
+      }, apiHost)
+        .then(res => {
+          if (res.status == 200) {
+            query.feedback = feedback;
+            setQueries((prev: Query[]) => {
+              return prev.map((q, i) => (i === index ? query : q));
+            });
+          }
+        })
+        .catch(err => console.log("Connection failed",err))
+    }
+    else {
+      delete query.feedback;
+      setQueries((prev: Query[]) => {
+        return prev.map((q, i) => (i === index ? query : q));
+      });
+
+    }
+  }
+
   async function stream(question: string) {
     setStatus('loading')
     try {
@@ -329,19 +436,24 @@ export const DocsGPTWidget = ({
           question: question,
           apiKey: apiKey,
           apiHost: apiHost,
-          selectedDocs: selectDocs,
           history: queries,
           conversationId: conversationId,
           onEvent: (event: MessageEvent) => {
             const data = JSON.parse(event.data);
             // check if the 'end' event has been received
             if (data.type === 'end') {
-              // set status to 'idle'
               setStatus('idle');
-
-            } else if (data.type === 'id') {
+            }
+            else if (data.type === 'id') {
               setConversationId(data.id)
-            } else {
+            }
+            else if (data.type === 'error') {
+              const updatedQueries = [...queries];
+              updatedQueries[updatedQueries.length - 1].error = data.error;
+              setQueries(updatedQueries);
+              setStatus('idle')
+            }
+            else {
               const result = data.answer;
               const streamingResponse = queries[queries.length - 1].response ? queries[queries.length - 1].response : '';
               const updatedQueries = [...queries];
@@ -353,7 +465,7 @@ export const DocsGPTWidget = ({
       );
     } catch (error) {
       const updatedQueries = [...queries];
-      updatedQueries[updatedQueries.length - 1].error = 'error'
+      updatedQueries[updatedQueries.length - 1].error = 'Something went wrong !'
       setQueries(updatedQueries);
       setStatus('idle')
       //setEventInterrupt(false)
@@ -372,16 +484,21 @@ export const DocsGPTWidget = ({
     event.currentTarget.src = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png";
   };
   return (
-    <>
-      <WidgetContainer>
+    <ThemeProvider theme={themes[theme]}>
+      {open && size === 'large' &&
+        <Overlay onClick={() => {
+          setOpen(false)
+        }} />
+      }
+      <FloatingButton bgcolor={buttonBg} onClick={() => setOpen(!open)} hidden={open}>
+        <img style={{ maxHeight: '4rem', maxWidth: '4rem' }} src={buttonIcon} />
+      </FloatingButton>
+      <WidgetContainer modal={size == 'large'}>
         <GlobalStyles />
-        {!open && <FloatingButton onClick={() => setOpen(true)} hidden={open}>
-          <MessageIcon style={{ marginTop: '8px' }} />
-        </FloatingButton>}
         {open && <StyledContainer>
           <div>
             <CancelButton onClick={() => setOpen(false)}>
-              <Cross2Icon width={24} height={24} color='white' />
+              <Cross2Icon width={24} height={24} color={theme === 'light' ? 'black' : 'white'} />
             </CancelButton>
             <Header>
               <IconWrapper>
@@ -393,7 +510,7 @@ export const DocsGPTWidget = ({
               </ContentWrapper>
             </Header>
           </div>
-          <Conversation onWheel={handleUserInterrupt} onTouchMove={handleUserInterrupt}>
+          <Conversation size={size} onWheel={handleUserInterrupt} onTouchMove={handleUserInterrupt}>
             {
               queries.length > 0 ? queries?.map((query, index) => {
                 return (
@@ -408,13 +525,34 @@ export const DocsGPTWidget = ({
                       </MessageBubble>
                     }
                     {
-                      query.response ? <MessageBubble type='ANSWER'>
+                      query.response ? <MessageBubble onMouseOver={() => { isBubbleHovered.current = true }} type='ANSWER'>
                         <Message
                           type='ANSWER'
                           ref={(index === queries.length - 1) ? endMessageRef : null}
                         >
-                          <div className="response" dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(snarkdown(query.response)) }} />
+                          <div
+                            className="response"
+                            dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(md.render(query.response)) }}
+                          />
                         </Message>
+
+                        {collectFeedback &&
+                          <Feedback>
+                            <Like
+                              style={{
+                                stroke: query.feedback == 'LIKE' ? '#8860DB' : '#c0c0c0',
+                                visibility: query.feedback == 'LIKE' ? 'visible' : 'hidden'
+                              }}
+                              fill='none'
+                              onClick={() => handleFeedback("LIKE", index)} />
+                            <Dislike
+                              style={{
+                                stroke: query.feedback == 'DISLIKE' ? '#ed8085' : '#c0c0c0',
+                                visibility: query.feedback == 'DISLIKE' ? 'visible' : 'hidden'
+                              }}
+                              fill='none'
+                              onClick={() => handleFeedback("DISLIKE", index)} />
+                          </Feedback>}
                       </MessageBubble>
                         : <div>
                           {
@@ -424,7 +562,7 @@ export const DocsGPTWidget = ({
                               </IconWrapper>
                               <div>
                                 <h5 style={{ margin: 2 }}>Network Error</h5>
-                                <span style={{ margin: 2, fontSize: '13px' }}>Something went wrong !</span>
+                                <span style={{ margin: 2, fontSize: '13px' }}>{query.error}</span>
                               </div>
                             </ErrorAlert>
                               : <MessageBubble type='ANSWER'>
@@ -439,22 +577,23 @@ export const DocsGPTWidget = ({
                     }
                   </React.Fragment>)
               })
-                : <Hero title={heroTitle} description={heroDescription} />
+                : <Hero title={heroTitle} description={heroDescription} theme={theme} />
             }
           </Conversation>
-
           <PromptContainer
+            size={size}
             onSubmit={handleSubmit}>
             <StyledInput
               value={prompt} onChange={(event) => setPrompt(event.target.value)}
               type='text' placeholder="What do you want to do?" />
             <StyledButton
-              disabled={prompt.length == 0 || status !== 'idle'}>
+              size={size}
+              disabled={prompt.trim().length == 0 || status !== 'idle'}>
               <PaperPlaneIcon width={15} height={15} color='white' />
             </StyledButton>
           </PromptContainer>
         </StyledContainer>}
       </WidgetContainer>
-    </>
+    </ThemeProvider>
   )
 }

extensions/react-widget/src/main.tsx

@@ -2,10 +2,11 @@ import React from 'react';
 import { createRoot } from 'react-dom/client';
 import { DocsGPTWidget } from './components/DocsGPTWidget';
 
-const renderWidget = (elementId: string, props = {}) => {
-  const root = createRoot(document.getElementById(elementId) as HTMLElement);
-  root.render(<DocsGPTWidget {...props} />);
-};
-
-(window as any).renderDocsGPTWidget = renderWidget;
+if (typeof window !== 'undefined') {
+  const renderWidget = (elementId: string, props = {}) => {
+    const root = createRoot(document.getElementById(elementId) as HTMLElement);
+    root.render(<DocsGPTWidget {...props} />);
+  };
+  (window as any).renderDocsGPTWidget = renderWidget;
+}
 export { DocsGPTWidget };

extensions/react-widget/src/requests/streamingApi.ts

@@ -1,92 +1,106 @@
+import { FEEDBACK } from "@/types";
 interface HistoryItem {
-    prompt: string;
-    response?: string;
-  }
+  prompt: string;
+  response?: string;
+}
 interface FetchAnswerStreamingProps {
-    question?: string;
-    apiKey?: string;
-    selectedDocs?: string;
-    history?: HistoryItem[];
-    conversationId?: string | null;
-    apiHost?: string;
-    onEvent?: (event: MessageEvent) => void;
-  }
+  question?: string;
+  apiKey?: string;
+  selectedDocs?: string;
+  history?: HistoryItem[];
+  conversationId?: string | null;
+  apiHost?: string;
+  onEvent?: (event: MessageEvent) => void;
+}
+interface FeedbackPayload {
+  question: string;
+  answer: string;
+  apikey: string;
+  feedback: FEEDBACK;
+}
 export function fetchAnswerStreaming({
-    question = '',
-    apiKey = '',
-    selectedDocs = '',
-    history = [],
-    conversationId = null,
-    apiHost = '',
-    onEvent = () => {console.log("Event triggered, but no handler provided.");}
-  }: FetchAnswerStreamingProps): Promise<void> {
-    let docPath = 'default';
-    if (selectedDocs) {
-      docPath = selectedDocs;
-    }
-  
-    return new Promise<void>((resolve, reject) => {
-      const body = {
-        question: question,
-        api_key: apiKey,
-        embeddings_key: apiKey,
-        active_docs: docPath,
-        history: JSON.stringify(history),
-        conversation_id: conversationId,
-        model: 'default'
-      };
-  
-      fetch(apiHost + '/stream', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify(body),
-      })
-        .then((response) => {
-          if (!response.body) throw Error('No response body');
-  
-          const reader = response.body.getReader();
-          const decoder = new TextDecoder('utf-8');
-          let counterrr = 0;
-          const processStream = ({
-            done,
-            value,
-          }: ReadableStreamReadResult<Uint8Array>) => {
-            if (done) {
-              resolve();
-              return;
+  question = '',
+  apiKey = '',
+  history = [],
+  conversationId = null,
+  apiHost = '',
+  onEvent = () => { console.log("Event triggered, but no handler provided."); }
+}: FetchAnswerStreamingProps): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const body = {
+      question: question,
+      history: JSON.stringify(history),
+      conversation_id: conversationId,
+      model: 'default',
+      api_key: apiKey
+    };
+    fetch(apiHost + '/stream', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(body),
+    })
+      .then((response) => {
+        if (!response.body) throw Error('No response body');
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder('utf-8');
+        let counterrr = 0;
+        const processStream = ({
+          done,
+          value,
+        }: ReadableStreamReadResult<Uint8Array>) => {
+          if (done) {
+            resolve();
+            return;
+          }
+
+          counterrr += 1;
+
+          const chunk = decoder.decode(value);
+
+          const lines = chunk.split('\n');
+
+          for (let line of lines) {
+            if (line.trim() == '') {
+              continue;
             }
-  
-            counterrr += 1;
-  
-            const chunk = decoder.decode(value);
-  
-            const lines = chunk.split('\n');
-  
-            for (let line of lines) {
-              if (line.trim() == '') {
-                continue;
-              }
-              if (line.startsWith('data:')) {
-                line = line.substring(5);
-              }
-  
-              const messageEvent = new MessageEvent('message', {
-                data: line,
-              });
-  
-              onEvent(messageEvent); // handle each message
+            if (line.startsWith('data:')) {
+              line = line.substring(5);
             }
-  
-            reader.read().then(processStream).catch(reject);
-          };
-  
+
+            const messageEvent = new MessageEvent('message', {
+              data: line,
+            });
+
+            onEvent(messageEvent); // handle each message
+          }
+
           reader.read().then(processStream).catch(reject);
-        })
-        .catch((error) => {
-          console.error('Connection failed:', error);
-          reject(error);
-        });
-    });
-  }
+        };
+
+        reader.read().then(processStream).catch(reject);
+      })
+      .catch((error) => {
+        console.error('Connection failed:', error);
+        reject(error);
+      });
+  });
+}
+
+
+export const sendFeedback = (payload: FeedbackPayload,apiHost:string): Promise<Response> => {
+  return fetch(`${apiHost}/api/feedback`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      question: payload.question,
+      answer: payload.answer,
+      feedback: payload.feedback,
+      api_key:payload.apikey
+    }),
+  });
+};

extensions/react-widget/src/types/index.ts

@@ -1,7 +1,7 @@
 export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER' | 'ERROR';
 export type Status = 'idle' | 'loading' | 'failed';
 export type FEEDBACK = 'LIKE' | 'DISLIKE';
-
+export type THEME = 'light' | 'dark';
 export interface Query {
   prompt: string;
   response?: string;
@@ -10,4 +10,18 @@ export interface Query {
   sources?: { title: string; text: string }[];
   conversationId?: string | null;
   title?: string | null;
+}
+export interface WidgetProps {
+  apiHost?: string;
+  apiKey?: string;
+  avatar?: string;
+  title?: string;
+  description?: string;
+  heroTitle?: string;
+  heroDescription?: string;
+  size?: 'small' | 'medium' | 'large';
+  theme?:THEME,
+  buttonIcon?:string;
+  buttonBg?:string;
+  collectFeedback?:boolean
 }

frontend/package.json

@@ -19,47 +19,49 @@
     ]
   },
   "dependencies": {
-    "@reduxjs/toolkit": "^1.9.2",
+    "@reduxjs/toolkit": "^2.2.7",
     "@vercel/analytics": "^0.1.10",
-    "i18next": "^23.11.5",
+    "chart.js": "^4.4.4",
+    "i18next": "^23.15.1",
     "i18next-browser-languagedetector": "^8.0.0",
     "prop-types": "^15.8.1",
     "react": "^18.2.0",
+    "react-chartjs-2": "^5.2.0",
     "react-copy-to-clipboard": "^5.1.0",
-    "react-dom": "^18.2.0",
+    "react-dom": "^18.3.1",
     "react-dropzone": "^14.2.3",
-    "react-i18next": "^14.1.2",
-    "react-markdown": "^8.0.7",
+    "react-i18next": "^15.0.2",
+    "react-markdown": "^9.0.1",
     "react-redux": "^8.0.5",
     "react-router-dom": "^6.8.1",
     "react-syntax-highlighter": "^15.5.0",
-    "remark-gfm": "^3.0.0"
+    "remark-gfm": "^4.0.0"
   },
   "devDependencies": {
     "@types/react": "^18.0.27",
-    "@types/react-dom": "^18.0.10",
-    "@types/react-syntax-highlighter": "^15.5.6",
+    "@types/react-dom": "^18.3.0",
+    "@types/react-syntax-highlighter": "^15.5.13",
     "@typescript-eslint/eslint-plugin": "^5.51.0",
-    "@typescript-eslint/parser": "^5.51.0",
-    "@vitejs/plugin-react": "^4.2.1",
+    "@typescript-eslint/parser": "^5.62.0",
+    "@vitejs/plugin-react": "^4.3.1",
     "autoprefixer": "^10.4.13",
-    "eslint": "^8.33.0",
-    "eslint-config-prettier": "^8.6.0",
+    "eslint": "^8.57.1",
+    "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard-with-typescript": "^34.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-n": "^15.6.1",
-    "eslint-plugin-prettier": "^4.2.1",
-    "eslint-plugin-promise": "^6.1.1",
-    "eslint-plugin-react": "^7.32.2",
+    "eslint-plugin-import": "^2.30.0",
+    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-prettier": "^5.2.1",
+    "eslint-plugin-promise": "^6.6.0",
+    "eslint-plugin-react": "^7.35.0",
     "eslint-plugin-unused-imports": "^2.0.0",
     "husky": "^8.0.0",
-    "lint-staged": "^13.1.1",
-    "postcss": "^8.4.31",
-    "prettier": "^2.8.4",
-    "prettier-plugin-tailwindcss": "^0.2.2",
-    "tailwindcss": "^3.2.4",
+    "lint-staged": "^15.2.10",
+    "postcss": "^8.4.41",
+    "prettier": "^3.3.3",
+    "prettier-plugin-tailwindcss": "^0.6.6",
+    "tailwindcss": "^3.4.11",
     "typescript": "^4.9.5",
-    "vite": "^5.0.13",
+    "vite": "^5.4.6",
     "vite-plugin-svgr": "^4.2.0"
   }
 }

frontend/src/App.tsx

@@ -1,5 +1,4 @@
 import { Routes, Route } from 'react-router-dom';
-import { useEffect } from 'react';
 import Navigation from './Navigation';
 import Conversation from './conversation/Conversation';
 import About from './About';
@@ -10,18 +9,19 @@ import { useState } from 'react';
 import Setting from './settings';
 import './locale/i18n';
 import { Outlet } from 'react-router-dom';
-import SharedConversation from './conversation/SharedConversation';
+import { SharedConversation } from './conversation/SharedConversation';
 import { useDarkTheme } from './hooks';
 inject();
 
 function MainLayout() {
   const { isMobile } = useMediaQuery();
   const [navOpen, setNavOpen] = useState(!isMobile);
+
   return (
-    <div className="dark:bg-raisin-black">
+    <div className="dark:bg-raisin-black relative h-screen overflow-auto">
       <Navigation navOpen={navOpen} setNavOpen={setNavOpen} />
       <div
-        className={`min-h-screen ${
+        className={`h-[calc(100dvh-64px)] sm:h-screen ${
           !isMobile
             ? `ml-0 ${!navOpen ? 'md:mx-auto lg:mx-auto' : 'md:ml-72'}`
             : 'ml-0 md:ml-16'
@@ -34,19 +34,9 @@ function MainLayout() {
 }
 
 export default function App() {
-  const [isDarkTheme] = useDarkTheme();
-  useEffect(() => {
-    localStorage.setItem('selectedTheme', isDarkTheme ? 'Dark' : 'Light');
-    if (isDarkTheme) {
-      document
-        .getElementById('root')
-        ?.classList.add('dark', 'dark:bg-raisin-black');
-    } else {
-      document.getElementById('root')?.classList.remove('dark');
-    }
-  }, [isDarkTheme]);
+  useDarkTheme();
   return (
-    <>
+    <div className="h-full relative overflow-auto">
       <Routes>
         <Route element={<MainLayout />}>
           <Route index element={<Conversation />} />
@@ -56,6 +46,6 @@ export default function App() {
         <Route path="/share/:identifier" element={<SharedConversation />} />
         <Route path="/*" element={<PageNotFound />} />
       </Routes>
-    </>
+    </div>
   );
 }

frontend/src/Hero.tsx

@@ -19,7 +19,7 @@ export default function Hero({
   }>;
   return (
     <div
-      className={`mt-16 mb-4 flex w-full flex-col justify-end text-black-1000 dark:text-bright-gray sm:w-full lg:mt-6`}
+      className={`pt-20 sm:pt-0 pb-6 sm:pb-12 flex h-full  w-full flex-col  text-black-1000 dark:text-bright-gray sm:w-full px-2 sm:px-0`}
     >
       <div className="flex h-full w-full flex-col items-center justify-center">
         <div className="flex items-center">

frontend/src/Navigation.tsx

@@ -1,46 +1,47 @@
 import { useEffect, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
 import { NavLink, useNavigate } from 'react-router-dom';
+
+import conversationService from './api/services/conversationService';
+import userService from './api/services/userService';
+import Add from './assets/add.svg';
 import DocsGPT3 from './assets/cute_docsgpt3.svg';
 import Discord from './assets/discord.svg';
 import Expand from './assets/expand.svg';
 import Github from './assets/github.svg';
 import Hamburger from './assets/hamburger.svg';
-import HamburgerDark from './assets/hamburger-dark.svg';
 import Info from './assets/info.svg';
 import SettingGear from './assets/settingGear.svg';
 import Twitter from './assets/TwitterX.svg';
-import Add from './assets/add.svg';
 import UploadIcon from './assets/upload.svg';
-import { ActiveState } from './models/misc';
-import APIKeyModal from './preferences/APIKeyModal';
-import DeleteConvModal from './modals/DeleteConvModal';
-
-import {
-  selectApiKeyStatus,
-  selectSelectedDocs,
-  selectSelectedDocsStatus,
-  selectSourceDocs,
-  setSelectedDocs,
-  selectConversations,
-  setConversations,
-  selectConversationId,
-  selectModalStateDeleteConv,
-  setModalStateDeleteConv,
-  setSourceDocs,
-} from './preferences/preferenceSlice';
+import SourceDropdown from './components/SourceDropdown';
 import {
   setConversation,
   updateConversationId,
 } from './conversation/conversationSlice';
-import { useMediaQuery, useOutsideAlerter } from './hooks';
-import Upload from './upload/Upload';
-import { Doc, getConversations, getDocs } from './preferences/preferenceApi';
-import SelectDocsModal from './preferences/SelectDocsModal';
 import ConversationTile from './conversation/ConversationTile';
-import { useDarkTheme } from './hooks';
-import SourceDropdown from './components/SourceDropdown';
-import { useTranslation } from 'react-i18next';
+import { useDarkTheme, useMediaQuery, useOutsideAlerter } from './hooks';
+import useDefaultDocument from './hooks/useDefaultDocument';
+import DeleteConvModal from './modals/DeleteConvModal';
+import { ActiveState, Doc } from './models/misc';
+import APIKeyModal from './preferences/APIKeyModal';
+import { getConversations, getDocs } from './preferences/preferenceApi';
+import {
+  selectApiKeyStatus,
+  selectConversationId,
+  selectConversations,
+  selectModalStateDeleteConv,
+  selectSelectedDocs,
+  selectSelectedDocsStatus,
+  selectSourceDocs,
+  setConversations,
+  setModalStateDeleteConv,
+  setSelectedDocs,
+  setSourceDocs,
+} from './preferences/preferenceSlice';
+import Upload from './upload/Upload';
+
 interface NavigationProps {
   navOpen: boolean;
   setNavOpen: React.Dispatch<React.SetStateAction<boolean>>;
@@ -85,7 +86,6 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
     useState<ActiveState>('INACTIVE');
 
   const navRef = useRef(null);
-  const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
 
   const navigate = useNavigate();
 
@@ -106,9 +106,8 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   }
 
   const handleDeleteAllConversations = () => {
-    fetch(`${apiHost}/api/delete_all_conversations`, {
-      method: 'POST',
-    })
+    conversationService
+      .deleteAll()
       .then(() => {
         fetchConversations();
       })
@@ -116,9 +115,8 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   };
 
   const handleDeleteConversation = (id: string) => {
-    fetch(`${apiHost}/api/delete_conversation?id=${id}`, {
-      method: 'POST',
-    })
+    conversationService
+      .delete(id, {})
       .then(() => {
         fetchConversations();
       })
@@ -126,19 +124,9 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   };
 
   const handleDeleteClick = (doc: Doc) => {
-    const docPath = `indexes/local/${doc.name}`;
-
-    fetch(`${apiHost}/api/delete_old?path=${docPath}`, {
-      method: 'GET',
-    })
+    userService
+      .deletePath(doc.id ?? '')
       .then(() => {
-        // remove the image element from the DOM
-        // const imageElement = document.querySelector(
-        //   `#img-${index}`,
-        // ) as HTMLElement;
-        // const parentElement = imageElement.parentNode as HTMLElement;
-        // parentElement.parentNode?.removeChild(parentElement);
-
         return getDocs();
       })
       .then((updatedDocs) => {
@@ -153,10 +141,8 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   };
 
   const handleConversationClick = (index: string) => {
-    // fetch the conversation from the server and setConversation in the store
-    fetch(`${apiHost}/api/get_single_conversation?id=${index}`, {
-      method: 'GET',
-    })
+    conversationService
+      .getConversation(index)
       .then((response) => response.json())
       .then((data) => {
         navigate('/');
@@ -173,13 +159,8 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
     name: string;
     id: string;
   }) {
-    await fetch(`${apiHost}/api/update_conversation_name`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify(updatedConversation),
-    })
+    await conversationService
+      .update(updatedConversation)
       .then((response) => response.json())
       .then((data) => {
         if (data) {
@@ -191,16 +172,12 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         console.error(err);
       });
   }
-  useOutsideAlerter(
-    navRef,
-    () => {
-      if (isMobile && navOpen && apiKeyModalState === 'INACTIVE') {
-        setNavOpen(false);
-        setIsDocsListOpen(false);
-      }
-    },
-    [navOpen, isDocsListOpen, apiKeyModalState],
-  );
+  useOutsideAlerter(navRef, () => {
+    if (isMobile && navOpen && apiKeyModalState === 'INACTIVE') {
+      setNavOpen(false);
+      setIsDocsListOpen(false);
+    }
+  }, [navOpen, isDocsListOpen, apiKeyModalState]);
 
   /*
     Needed to fix bug where if mobile nav was closed and then window was resized to desktop, nav would still be closed but the button to open would be gone, as per #1 on issue #146
@@ -209,6 +186,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   useEffect(() => {
     setNavOpen(!isMobile);
   }, [isMobile]);
+  useDefaultDocument();
   return (
     <>
       {!navOpen && (
@@ -403,24 +381,18 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
           </div>
         </div>
       </div>
-      <div className="fixed z-10 h-16 w-full border-b-2 bg-gray-50 dark:border-b-purple-taupe dark:bg-chinese-black md:hidden">
+      <div className="sticky z-10 h-16 w-full border-b-2 bg-gray-50 dark:border-b-purple-taupe dark:bg-chinese-black md:hidden">
         <button
           className="mt-5 ml-6 h-6 w-6 md:hidden"
           onClick={() => setNavOpen(true)}
         >
           <img
-            src={isDarkTheme ? HamburgerDark : Hamburger}
+            src={Hamburger}
             alt="menu toggle"
-            className="w-7"
+            className="w-7 filter dark:invert"
           />
         </button>
       </div>
-
-      <SelectDocsModal
-        modalState={selectedDocsModalState}
-        setModalState={setSelectedDocsModalState}
-        isCancellable={isSelectedDocsSet}
-      />
       <APIKeyModal
         modalState={apiKeyModalState}
         setModalState={setApiKeyModalState}

frontend/src/api/client.ts

@@ -0,0 +1,69 @@
+const baseURL = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+
+const defaultHeaders = {
+  'Content-Type': 'application/json',
+};
+
+const apiClient = {
+  get: (url: string, headers = {}, signal?: AbortSignal): Promise<any> =>
+    fetch(`${baseURL}${url}`, {
+      method: 'GET',
+      headers: {
+        ...defaultHeaders,
+        ...headers,
+      },
+      signal,
+    }).then((response) => {
+      return response;
+    }),
+
+  post: (
+    url: string,
+    data: any,
+    headers = {},
+    signal?: AbortSignal,
+  ): Promise<any> =>
+    fetch(`${baseURL}${url}`, {
+      method: 'POST',
+      headers: {
+        ...defaultHeaders,
+        ...headers,
+      },
+      body: JSON.stringify(data),
+      signal,
+    }).then((response) => {
+      return response;
+    }),
+
+  put: (
+    url: string,
+    data: any,
+    headers = {},
+    signal?: AbortSignal,
+  ): Promise<any> =>
+    fetch(`${baseURL}${url}`, {
+      method: 'PUT',
+      headers: {
+        ...defaultHeaders,
+        ...headers,
+      },
+      body: JSON.stringify(data),
+      signal,
+    }).then((response) => {
+      return response;
+    }),
+
+  delete: (url: string, headers = {}, signal?: AbortSignal): Promise<any> =>
+    fetch(`${baseURL}${url}`, {
+      method: 'DELETE',
+      headers: {
+        ...defaultHeaders,
+        ...headers,
+      },
+      signal,
+    }).then((response) => {
+      return response;
+    }),
+};
+
+export default apiClient;

frontend/src/api/endpoints.ts

@@ -0,0 +1,38 @@
+const endpoints = {
+  USER: {
+    DOCS: '/api/combine',
+    DOCS_CHECK: '/api/docs_check',
+    API_KEYS: '/api/get_api_keys',
+    CREATE_API_KEY: '/api/create_api_key',
+    DELETE_API_KEY: '/api/delete_api_key',
+    PROMPTS: '/api/get_prompts',
+    CREATE_PROMPT: '/api/create_prompt',
+    DELETE_PROMPT: '/api/delete_prompt',
+    UPDATE_PROMPT: '/api/update_prompt',
+    SINGLE_PROMPT: (id: string) => `/api/get_single_prompt?id=${id}`,
+    DELETE_PATH: (docPath: string) => `/api/delete_old?source_id=${docPath}`,
+    TASK_STATUS: (task_id: string) => `/api/task_status?task_id=${task_id}`,
+    MESSAGE_ANALYTICS: '/api/get_message_analytics',
+    TOKEN_ANALYTICS: '/api/get_token_analytics',
+    FEEDBACK_ANALYTICS: '/api/get_feedback_analytics',
+    LOGS: `/api/get_user_logs`,
+    MANAGE_SYNC: '/api/manage_sync',
+  },
+  CONVERSATION: {
+    ANSWER: '/api/answer',
+    ANSWER_STREAMING: '/stream',
+    SEARCH: '/api/search',
+    FEEDBACK: '/api/feedback',
+    CONVERSATION: (id: string) => `/api/get_single_conversation?id=${id}`,
+    CONVERSATIONS: '/api/get_conversations',
+    SHARE_CONVERSATION: (isPromptable: boolean) =>
+      `/api/share?isPromptable=${isPromptable}`,
+    SHARED_CONVERSATION: (identifier: string) =>
+      `/api/shared_conversation/${identifier}`,
+    DELETE: (id: string) => `/api/delete_conversation?id=${id}`,
+    DELETE_ALL: '/api/delete_all_conversations',
+    UPDATE: '/api/update_conversation_name',
+  },
+};
+
+export default endpoints;

frontend/src/api/services/conversationService.ts

@@ -0,0 +1,32 @@
+import apiClient from '../client';
+import endpoints from '../endpoints';
+
+const conversationService = {
+  answer: (data: any, signal: AbortSignal): Promise<any> =>
+    apiClient.post(endpoints.CONVERSATION.ANSWER, data, {}, signal),
+  answerStream: (data: any, signal: AbortSignal): Promise<any> =>
+    apiClient.post(endpoints.CONVERSATION.ANSWER_STREAMING, data, {}, signal),
+  search: (data: any): Promise<any> =>
+    apiClient.post(endpoints.CONVERSATION.SEARCH, data),
+  feedback: (data: any): Promise<any> =>
+    apiClient.post(endpoints.CONVERSATION.FEEDBACK, data),
+  getConversation: (id: string): Promise<any> =>
+    apiClient.get(endpoints.CONVERSATION.CONVERSATION(id)),
+  getConversations: (): Promise<any> =>
+    apiClient.get(endpoints.CONVERSATION.CONVERSATIONS),
+  shareConversation: (isPromptable: boolean, data: any): Promise<any> =>
+    apiClient.post(
+      endpoints.CONVERSATION.SHARE_CONVERSATION(isPromptable),
+      data,
+    ),
+  getSharedConversation: (identifier: string): Promise<any> =>
+    apiClient.get(endpoints.CONVERSATION.SHARED_CONVERSATION(identifier)),
+  delete: (id: string, data: any): Promise<any> =>
+    apiClient.post(endpoints.CONVERSATION.DELETE(id), data),
+  deleteAll: (): Promise<any> =>
+    apiClient.get(endpoints.CONVERSATION.DELETE_ALL),
+  update: (data: any): Promise<any> =>
+    apiClient.post(endpoints.CONVERSATION.UPDATE, data),
+};
+
+export default conversationService;

frontend/src/api/services/userService.ts

@@ -0,0 +1,38 @@
+import apiClient from '../client';
+import endpoints from '../endpoints';
+
+const userService = {
+  getDocs: (): Promise<any> => apiClient.get(endpoints.USER.DOCS),
+  checkDocs: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.DOCS_CHECK, data),
+  getAPIKeys: (): Promise<any> => apiClient.get(endpoints.USER.API_KEYS),
+  createAPIKey: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.CREATE_API_KEY, data),
+  deleteAPIKey: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.DELETE_API_KEY, data),
+  getPrompts: (): Promise<any> => apiClient.get(endpoints.USER.PROMPTS),
+  createPrompt: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.CREATE_PROMPT, data),
+  deletePrompt: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.DELETE_PROMPT, data),
+  updatePrompt: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.UPDATE_PROMPT, data),
+  getSinglePrompt: (id: string): Promise<any> =>
+    apiClient.get(endpoints.USER.SINGLE_PROMPT(id)),
+  deletePath: (docPath: string): Promise<any> =>
+    apiClient.get(endpoints.USER.DELETE_PATH(docPath)),
+  getTaskStatus: (task_id: string): Promise<any> =>
+    apiClient.get(endpoints.USER.TASK_STATUS(task_id)),
+  getMessageAnalytics: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.MESSAGE_ANALYTICS, data),
+  getTokenAnalytics: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.TOKEN_ANALYTICS, data),
+  getFeedbackAnalytics: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.FEEDBACK_ANALYTICS, data),
+  getLogs: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.LOGS, data),
+  manageSync: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.MANAGE_SYNC, data),
+};
+
+export default userService;

frontend/src/assets/chevron-right.svg

@@ -0,0 +1,3 @@
+<svg width="7" height="12" viewBox="0 0 7 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.29154 4.88202L1.70154 0.29202C1.60896 0.199438 1.49905 0.125998 1.37808 0.0758932C1.25712 0.0257882 1.12747 -2.37536e-07 0.99654 -2.44235e-07C0.86561 -2.50934e-07 0.735961 0.0257882 0.614997 0.0758931C0.494033 0.125998 0.384122 0.199438 0.29154 0.29202C0.198958 0.384602 0.125519 0.494513 0.0754137 0.615477C0.0253086 0.736441 -0.00048069 0.86609 -0.000480695 0.99702C-0.000480701 1.12795 0.0253086 1.2576 0.0754136 1.37856C0.125519 1.49953 0.198958 1.60944 0.29154 1.70202L4.17154 5.59202L0.29154 9.47202C0.198958 9.5646 0.125518 9.67451 0.0754133 9.79548C0.0253082 9.91644 -0.000481091 10.0461 -0.000481097 10.177C-0.000481102 10.3079 0.0253082 10.4376 0.0754132 10.5586C0.125518 10.6795 0.198958 10.7894 0.29154 10.882C0.384121 10.9746 0.494032 11.048 0.614996 11.0981C0.73596 11.1483 0.865609 11.174 0.99654 11.174C1.12747 11.174 1.25712 11.1483 1.37808 11.0981C1.49905 11.048 1.60896 10.9746 1.70154 10.882L6.29154 6.29202C6.38424 6.19951 6.45779 6.08962 6.50797 5.96864C6.55815 5.84767 6.58398 5.71799 6.58398 5.58702C6.58398 5.45605 6.55815 5.32637 6.50797 5.2054C6.45779 5.08442 6.38424 4.97453 6.29154 4.88202Z" fill="#666666"/>
+</svg>

frontend/src/assets/document.svg

@@ -0,0 +1,3 @@
+<svg width="14" height="15" viewBox="0 0 14 15" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M12.9294 5.375V12.4583C12.9294 12.8341 12.7801 13.1944 12.5145 13.4601C12.2488 13.7257 11.8885 13.875 11.5127 13.875H3.01274C2.63701 13.875 2.27668 13.7257 2.011 13.4601C1.74532 13.1944 1.59607 12.8341 1.59607 12.4583V2.54167C1.59607 2.16594 1.74532 1.80561 2.011 1.53993C2.27668 1.27426 2.63701 1.125 3.01274 1.125H8.6794M12.9294 5.375V5.25317C12.9293 4.87747 12.78 4.5172 12.5143 4.25158L9.80282 1.54008C9.53721 1.27439 9.17693 1.12508 8.80124 1.125H8.6794M12.9294 5.375H10.0961C9.72035 5.375 9.36001 5.22574 9.09434 4.96007C8.82866 4.69439 8.6794 4.33406 8.6794 3.95833V1.125" stroke="#949494" stroke-width="1.41667" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

frontend/src/assets/sources.svg

@@ -0,0 +1,25 @@
+<svg width="24" height="27" viewBox="0 0 24 27" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M1.73517 17.2121L0.538918 17.9125C0.3753 18.0084 0.239616 18.1455 0.145332 18.31C0.0510491 18.4746 0.00144577 18.661 0.00144577 18.8506C0.00144577 19.0403 0.0510491 19.2267 0.145332 19.3912C0.239616 19.5558 0.3753 19.6929 0.538918 19.7888L11.0514 25.9513C11.2181 26.0489 11.4078 26.1004 11.601 26.1004C11.7941 26.1004 11.9838 26.0489 12.1505 25.9513L22.663 19.7888C22.8266 19.6929 22.9623 19.5558 23.0566 19.3912C23.1509 19.2267 23.2005 19.0403 23.2005 18.8506C23.2005 18.661 23.1509 18.4746 23.0566 18.31C22.9623 18.1455 22.8266 18.0084 22.663 17.9125L21.4668 17.2107L13.2511 22.0276C12.7506 22.321 12.1811 22.4757 11.601 22.4757C11.0209 22.4757 10.4513 22.321 9.95087 22.0276L1.73517 17.2121Z" fill="url(#paint0_linear_4013_8178)"/>
+<path d="M1.73517 11.4121L0.538918 12.1124C0.3753 12.2084 0.239616 12.3454 0.145332 12.51C0.0510491 12.6746 0.00144577 12.8609 0.00144577 13.0506C0.00144577 13.2403 0.0510491 13.4266 0.145332 13.5912C0.239616 13.7558 0.3753 13.8928 0.538918 13.9887L11.0514 20.1512C11.2181 20.2489 11.4078 20.3003 11.601 20.3003C11.7941 20.3003 11.9838 20.2489 12.1505 20.1512L22.663 13.9887C22.8266 13.8928 22.9623 13.7558 23.0566 13.5912C23.1509 13.4266 23.2005 13.2403 23.2005 13.0506C23.2005 12.8609 23.1509 12.6746 23.0566 12.51C22.9623 12.3454 22.8266 12.2084 22.663 12.1124L21.4668 11.4106L13.2511 16.2275C12.7506 16.5209 12.1811 16.6756 11.601 16.6756C11.0209 16.6756 10.4513 16.5209 9.95087 16.2275L1.73517 11.4121Z" fill="url(#paint1_linear_4013_8178)"/>
+<path d="M12.152 0.149921C11.9849 0.0517579 11.7947 0 11.601 0C11.4072 0 11.217 0.0517579 11.05 0.149921L0.537472 6.31242C0.373854 6.40835 0.23817 6.5454 0.143887 6.70997C0.0496035 6.87454 0 7.06091 0 7.25057C0 7.44024 0.0496035 7.6266 0.143887 7.79117C0.23817 7.95574 0.373854 8.09279 0.537472 8.18872L11.05 14.3512C11.217 14.4494 11.4072 14.5011 11.601 14.5011C11.7947 14.5011 11.9849 14.4494 12.152 14.3512L22.6645 8.18872C22.8281 8.09279 22.9638 7.95574 23.0581 7.79117C23.1523 7.6266 23.2019 7.44024 23.2019 7.25057C23.2019 7.06091 23.1523 6.87454 23.0581 6.70997C22.9638 6.5454 22.8281 6.40835 22.6645 6.31242L12.152 0.149921Z" fill="url(#paint2_linear_4013_8178)"/>
+<defs>
+<linearGradient id="paint0_linear_4013_8178" x1="0.00144577" y1="21.6555" x2="23.2005" y2="21.6555" gradientUnits="userSpaceOnUse">
+<stop stop-color="#70FDF7"/>
+<stop offset="0.325" stop-color="#747696"/>
+<stop offset="0.68" stop-color="#BD5372"/>
+<stop offset="1" stop-color="#F5A06C"/>
+</linearGradient>
+<linearGradient id="paint1_linear_4013_8178" x1="0.00144577" y1="15.8555" x2="23.2005" y2="15.8555" gradientUnits="userSpaceOnUse">
+<stop stop-color="#70FDF7"/>
+<stop offset="0.325" stop-color="#747696"/>
+<stop offset="0.68" stop-color="#BD5372"/>
+<stop offset="1" stop-color="#F5A06C"/>
+</linearGradient>
+<linearGradient id="paint2_linear_4013_8178" x1="0" y1="7.25057" x2="23.2019" y2="7.25057" gradientUnits="userSpaceOnUse">
+<stop stop-color="#70FDF7"/>
+<stop offset="0.325" stop-color="#747696"/>
+<stop offset="0.68" stop-color="#BD5372"/>
+<stop offset="1" stop-color="#F5A06C"/>
+</linearGradient>
+</defs>
+</svg>

frontend/src/assets/sync.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="24px" viewBox="0 0 24 24" width="24px" fill="#ffffff"><path d="M.01 0h24v24h-24V0z" fill="none"/><path d="M12 4V1L8 5l4 4V6c3.31 0 6 2.69 6 6 0 1.01-.25 1.97-.7 2.8l1.46 1.46C19.54 15.03 20 13.57 20 12c0-4.42-3.58-8-8-8zm0 14c-3.31 0-6-2.69-6-6 0-1.01.25-1.97.7-2.8L5.24 7.74C4.46 8.97 4 10.43 4 12c0 4.42 3.58 8 8 8v3l4-4-4-4v3z"/></svg>

frontend/src/components/CopyButton.tsx

@@ -1,16 +1,24 @@
-import { useState } from 'react';
-import Copy from './../assets/copy.svg?react';
-import CheckMark from './../assets/checkmark.svg?react';
 import copy from 'copy-to-clipboard';
+import { useState } from 'react';
 
-export default function CoppyButton({ text }: { text: string }) {
+import CheckMark from '../assets/checkmark.svg?react';
+import Copy from '../assets/copy.svg?react';
+
+export default function CoppyButton({
+  text,
+  colorLight,
+  colorDark,
+}: {
+  text: string;
+  colorLight?: string;
+  colorDark?: string;
+}) {
   const [copied, setCopied] = useState(false);
   const [isCopyHovered, setIsCopyHovered] = useState(false);
 
   const handleCopyClick = (text: string) => {
     copy(text);
     setCopied(true);
-    // Reset copied to false after a few seconds
     setTimeout(() => {
       setCopied(false);
     }, 3000);
@@ -20,8 +28,8 @@ export default function CoppyButton({ text }: { text: string }) {
     <div
       className={`flex items-center justify-center rounded-full p-2 ${
         isCopyHovered
-          ? 'bg-[#EEEEEE] dark:bg-purple-taupe'
-          : 'bg-[#ffffff] dark:bg-transparent'
+          ? `bg-[#EEEEEE] dark:bg-purple-taupe`
+          : `bg-[${colorLight ? colorLight : '#FFFFFF'}] dark:bg-[${colorDark ? colorDark : 'transparent'}]`
       }`}
     >
       {copied ? (

frontend/src/components/Dropdown.tsx

@@ -16,6 +16,7 @@ function Dropdown({
   showDelete,
   onDelete,
   placeholder,
+  contentSize = 'text-base',
 }: {
   options:
     | string[]
@@ -26,6 +27,7 @@ function Dropdown({
     | string
     | { label: string; value: string }
     | { value: number; description: string }
+    | { name: string; id: string; type: string }
     | null;
   onSelect:
     | ((value: string) => void)
@@ -41,6 +43,7 @@ function Dropdown({
   showDelete?: boolean;
   onDelete?: (value: string) => void;
   placeholder?: string;
+  contentSize?: string;
 }) {
   const dropdownRef = React.useRef<HTMLDivElement>(null);
   const [isOpen, setIsOpen] = React.useState(false);
@@ -79,26 +82,26 @@ function Dropdown({
         }`}
       >
         {typeof selectedValue === 'string' ? (
-          <span className="overflow-hidden text-ellipsis dark:text-bright-gray">
+          <span className="truncate dark:text-bright-gray">
             {selectedValue}
           </span>
         ) : (
           <span
-            className={`overflow-hidden text-ellipsis dark:text-bright-gray ${
+            className={`truncate dark:text-bright-gray ${
               !selectedValue && 'text-silver dark:text-gray-400'
-            }`}
+            } ${contentSize}`}
           >
             {selectedValue && 'label' in selectedValue
               ? selectedValue.label
               : selectedValue && 'description' in selectedValue
-              ? `${
-                  selectedValue.value < 1e9
-                    ? selectedValue.value + ` (${selectedValue.description})`
-                    : selectedValue.description
-                }`
-              : placeholder
-              ? placeholder
-              : 'From URL'}
+                ? `${
+                    selectedValue.value < 1e9
+                      ? selectedValue.value + ` (${selectedValue.description})`
+                      : selectedValue.description
+                  }`
+                : placeholder
+                  ? placeholder
+                  : 'From URL'}
           </span>
         )}
         <img
@@ -123,19 +126,19 @@ function Dropdown({
                   onSelect(option);
                   setIsOpen(false);
                 }}
-                className="ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3 dark:text-light-gray"
+                className={`ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3 dark:text-light-gray ${contentSize}`}
               >
                 {typeof option === 'string'
                   ? option
                   : option.name
-                  ? option.name
-                  : option.label
-                  ? option.label
-                  : `${
-                      option.value < 1e9
-                        ? option.value + ` (${option.description})`
-                        : option.description
-                    }`}
+                    ? option.name
+                    : option.label
+                      ? option.label
+                      : `${
+                          option.value < 1e9
+                            ? option.value + ` (${option.description})`
+                            : option.description
+                        }`}
               </span>
               {showEdit && onEdit && (
                 <img

frontend/src/components/DropdownMenu.tsx

@@ -0,0 +1,88 @@
+import React from 'react';
+
+type DropdownMenuProps = {
+  name: string;
+  options: { label: string; value: string }[];
+  onSelect: (value: string) => void;
+  defaultValue?: string;
+  icon?: string;
+};
+
+export default function DropdownMenu({
+  name,
+  options,
+  onSelect,
+  defaultValue = 'none',
+  icon,
+}: DropdownMenuProps) {
+  const dropdownRef = React.useRef<HTMLDivElement>(null);
+  const [isOpen, setIsOpen] = React.useState(false);
+  const [selectedOption, setSelectedOption] = React.useState(
+    options.find((option) => option.value === defaultValue) || options[0],
+  );
+
+  const handleToggle = () => {
+    setIsOpen(!isOpen);
+  };
+  const handleClickOutside = (event: MouseEvent) => {
+    if (
+      dropdownRef.current &&
+      !dropdownRef.current.contains(event.target as Node)
+    ) {
+      setIsOpen(false);
+    }
+  };
+  const handleClickOption = (optionId: number) => {
+    setIsOpen(false);
+    setSelectedOption(options[optionId]);
+    onSelect(options[optionId].value);
+  };
+
+  React.useEffect(() => {
+    document.addEventListener('mousedown', handleClickOutside);
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+    };
+  }, []);
+  return (
+    <div className="static inline-block text-left" ref={dropdownRef}>
+      <button
+        onClick={handleToggle}
+        className="flex w-20 cursor-pointer flex-row items-center gap-px rounded-3xl border-purple-30/25 bg-purple-30 p-2 text-xs text-white hover:bg-[#6F3FD1] focus:outline-none"
+      >
+        {icon && <img src={icon} alt="OptionIcon" className="h-4 w-4" />}
+        {selectedOption.value !== 'never' ? selectedOption.label : name}
+      </button>
+      <div
+        className={`absolute z-50 right-0 mt-1 w-28 transform rounded-md bg-transparent shadow-lg ring-1 ring-black ring-opacity-5 transition-all duration-200 ease-in-out ${
+          isOpen
+            ? 'scale-100 opacity-100'
+            : 'pointer-events-none scale-95 opacity-0'
+        }`}
+      >
+        <div
+          role="menu"
+          className="overflow-hidden rounded-md"
+          aria-orientation="vertical"
+          aria-labelledby="options-menu"
+        >
+          {options.map((option, idx) => (
+            <div
+              id={`option-${idx}`}
+              className={`cursor-pointer px-4 py-2 text-xs hover:bg-gray-100 dark:text-light-gray dark:hover:bg-purple-taupe ${
+                selectedOption.value === option.value
+                  ? 'bg-gray-100 dark:bg-purple-taupe'
+                  : 'bg-white dark:bg-dark-charcoal'
+              }`}
+              role="menuitem"
+              key={option.value}
+              onClick={() => handleClickOption(idx)}
+            >
+              {option.label}
+            </div>
+          ))}
+        </div>
+      </div>
+    </div>
+  );
+}

frontend/src/components/Input.tsx

@@ -10,6 +10,7 @@ const Input = ({
   maxLength,
   className,
   colorVariant = 'silver',
+  borderVariant = 'thick',
   children,
   onChange,
   onPaste,
@@ -20,10 +21,13 @@ const Input = ({
     jet: 'border-jet',
     gray: 'border-gray-5000 dark:text-silver',
   };
-
+  const borderStyles = {
+    thin: 'border',
+    thick: 'border-2',
+  };
   return (
     <input
-      className={`h-[42px] w-full rounded-full border-2 px-3 outline-none dark:bg-transparent dark:text-white ${className} ${colorStyles[colorVariant]}`}
+      className={`h-[42px] w-full rounded-full px-3 py-1 outline-none dark:bg-transparent dark:text-white ${className} ${colorStyles[colorVariant]} ${borderStyles[borderVariant]}`}
       type={type}
       id={id}
       name={name}

frontend/src/components/Sidebar.tsx

@@ -0,0 +1,54 @@
+import React from 'react';
+
+import Exit from '../assets/exit.svg';
+
+type SidebarProps = {
+  isOpen: boolean;
+  toggleState: (arg0: boolean) => void;
+  children: React.ReactNode;
+};
+
+export default function Sidebar({
+  isOpen,
+  toggleState,
+  children,
+}: SidebarProps) {
+  const sidebarRef = React.useRef<HTMLDivElement>(null);
+
+  const handleClickOutside = (event: MouseEvent) => {
+    if (
+      sidebarRef.current &&
+      !sidebarRef.current.contains(event.target as Node)
+    ) {
+      toggleState(false);
+    }
+  };
+
+  React.useEffect(() => {
+    document.addEventListener('mousedown', handleClickOutside);
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+    };
+  }, []);
+  return (
+    <div ref={sidebarRef} className="h-vh relative">
+      <div
+        className={`fixed right-0 top-0 z-50 h-full w-72 transform bg-white shadow-xl transition-all duration-300 dark:bg-chinese-black sm:w-96 ${
+          isOpen ? 'translate-x-[10px]' : 'translate-x-full'
+        } border-l border-[#9ca3af]/10`}
+      >
+        <div className="flex w-full flex-row items-end justify-end px-4 pt-3">
+          <button
+            className="w-7 rounded-full p-2 hover:bg-gray-1000 hover:dark:bg-gun-metal"
+            onClick={() => toggleState(!isOpen)}
+          >
+            <img className="filter dark:invert" src={Exit} />
+          </button>
+        </div>
+        <div className="flex h-full flex-col items-center gap-2 py-4 px-6 text-center">
+          {children}
+        </div>
+      </div>
+    </div>
+  );
+}

frontend/src/components/SourceDropdown.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import Trash from '../assets/trash.svg';
 import Arrow2 from '../assets/dropdown-arrow.svg';
-import { Doc } from '../preferences/preferenceApi';
+import { Doc } from '../models/misc';
 import { useDispatch } from 'react-redux';
 import { useTranslation } from 'react-i18next';
 type Props = {
@@ -63,9 +63,6 @@ function SourceDropdown({
             <p className="max-w-3/4 truncate whitespace-nowrap">
               {selectedDocs?.name || 'None'}
             </p>
-            <p className="flex flex-col items-center justify-center">
-              {selectedDocs?.version}
-            </p>
           </div>
         </span>
         <img
@@ -77,7 +74,7 @@ function SourceDropdown({
         />
       </button>
       {isDocsListOpen && (
-        <div className="absolute left-0 right-0 z-50 -mt-1 max-h-40 overflow-y-auto rounded-b-xl border border-silver bg-white shadow-lg dark:border-silver/40 dark:bg-dark-charcoal">
+        <div className="absolute left-0 right-0 z-50 -mt-1 max-h-28 overflow-y-auto rounded-b-xl border border-silver bg-white shadow-lg dark:border-silver/40 dark:bg-dark-charcoal">
           {options ? (
             options.map((option: any, index: number) => {
               if (option.model === embeddingsName) {

frontend/src/components/types/index.ts

@@ -2,6 +2,7 @@ export type InputProps = {
   type: 'text' | 'number';
   value: string | string[] | number;
   colorVariant?: 'silver' | 'jet' | 'gray';
+  borderVariant?: 'thin' | 'thick';
   isAutoFocused?: boolean;
   id?: string;
   maxLength?: number;

frontend/src/conversation/Conversation.tsx

@@ -1,9 +1,22 @@
 import { Fragment, useEffect, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
-import { useDarkTheme } from '../hooks';
+
+import ArrowDown from '../assets/arrow-down.svg';
+import Send from '../assets/send.svg';
+import SendDark from '../assets/send_dark.svg';
+import ShareIcon from '../assets/share.svg';
+import SpinnerDark from '../assets/spinner-dark.svg';
+import Spinner from '../assets/spinner.svg';
+import RetryIcon from '../components/RetryIcon';
 import Hero from '../Hero';
+import { useDarkTheme } from '../hooks';
+import { ShareConversationModal } from '../modals/ShareConversationModal';
+import { selectConversationId } from '../preferences/preferenceSlice';
 import { AppDispatch } from '../store';
 import ConversationBubble from './ConversationBubble';
+import { handleSendFeedback } from './conversationHandlers';
+import { FEEDBACK, Query } from './conversationModels';
 import {
   addQuery,
   fetchAnswer,
@@ -11,26 +24,14 @@ import {
   selectStatus,
   updateQuery,
 } from './conversationSlice';
-import { selectConversationId } from '../preferences/preferenceSlice';
-import Send from './../assets/send.svg';
-import SendDark from './../assets/send_dark.svg';
-import Spinner from './../assets/spinner.svg';
-import SpinnerDark from './../assets/spinner-dark.svg';
-import { FEEDBACK, Query } from './conversationModels';
-import { sendFeedback } from './conversationApi';
-import { useTranslation } from 'react-i18next';
-import ArrowDown from './../assets/arrow-down.svg';
-import RetryIcon from '../components/RetryIcon';
-import ShareIcon from '../assets/share.svg';
-import { ShareConversationModal } from '../modals/ShareConversationModal';
 
 export default function Conversation() {
   const queries = useSelector(selectQueries);
   const status = useSelector(selectStatus);
   const conversationId = useSelector(selectConversationId);
   const dispatch = useDispatch<AppDispatch>();
-  const endMessageRef = useRef<HTMLDivElement>(null);
-  const inputRef = useRef<HTMLDivElement>(null);
+  const conversationRef = useRef<HTMLDivElement>(null);
+  const inputRef = useRef<HTMLTextAreaElement>(null);
   const [isDarkTheme] = useDarkTheme();
   const [hasScrolledToLast, setHasScrolledToLast] = useState(true);
   const fetchStream = useRef<any>(null);
@@ -47,39 +48,15 @@ export default function Conversation() {
   }, [queries.length, queries[queries.length - 1]]);
 
   useEffect(() => {
-    const element = document.getElementById('inputbox') as HTMLInputElement;
+    const element = document.getElementById('inputbox') as HTMLTextAreaElement;
     if (element) {
       element.focus();
     }
   }, []);
 
   useEffect(() => {
-    return () => {
-      if (status !== 'idle') {
-        fetchStream.current && fetchStream.current.abort(); //abort previous stream
-      }
-    };
-  }, [status]);
-
-  useEffect(() => {
-    const observerCallback: IntersectionObserverCallback = (entries) => {
-      entries.forEach((entry) => {
-        setHasScrolledToLast(entry.isIntersecting);
-      });
-    };
-
-    const observer = new IntersectionObserver(observerCallback, {
-      root: null,
-      threshold: [1, 0.8],
-    });
-    if (endMessageRef.current) {
-      observer.observe(endMessageRef.current);
-    }
-
-    return () => {
-      observer.disconnect();
-    };
-  }, [endMessageRef.current]);
+    fetchStream.current && fetchStream.current.abort();
+  }, [conversationId]);
 
   useEffect(() => {
     if (queries.length) {
@@ -89,10 +66,16 @@ export default function Conversation() {
   }, [queries[queries.length - 1]]);
 
   const scrollIntoView = () => {
-    endMessageRef?.current?.scrollIntoView({
-      behavior: 'smooth',
-      block: 'start',
-    });
+    if (!conversationRef?.current || eventInterrupt) return;
+
+    if (status === 'idle' || !queries[queries.length - 1].response) {
+      conversationRef.current.scrollTo({
+        behavior: 'smooth',
+        top: conversationRef.current.scrollHeight,
+      });
+    } else {
+      conversationRef.current.scrollTop = conversationRef.current.scrollHeight;
+    }
   };
 
   const handleQuestion = ({
@@ -112,20 +95,20 @@ export default function Conversation() {
   const handleFeedback = (query: Query, feedback: FEEDBACK, index: number) => {
     const prevFeedback = query.feedback;
     dispatch(updateQuery({ index, query: { feedback } }));
-    sendFeedback(query.prompt, query.response!, feedback).catch(() =>
+    handleSendFeedback(query.prompt, query.response!, feedback).catch(() =>
       dispatch(updateQuery({ index, query: { feedback: prevFeedback } })),
     );
   };
 
   const handleQuestionSubmission = () => {
-    if (inputRef.current?.textContent && status !== 'loading') {
+    if (inputRef.current?.value && status !== 'loading') {
       if (lastQueryReturnedErr) {
         // update last failed query with new prompt
         dispatch(
           updateQuery({
             index: queries.length - 1,
             query: {
-              prompt: inputRef.current.textContent,
+              prompt: inputRef.current.value,
             },
           }),
         );
@@ -134,9 +117,10 @@ export default function Conversation() {
           isRetry: true,
         });
       } else {
-        handleQuestion({ question: inputRef.current.textContent });
+        handleQuestion({ question: inputRef.current.value });
       }
-      inputRef.current.textContent = '';
+      inputRef.current.value = '';
+      handleInput();
     }
   };
 
@@ -145,7 +129,6 @@ export default function Conversation() {
     if (query.response) {
       responseView = (
         <ConversationBubble
-          ref={endMessageRef}
           className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'}`}
           key={`${index}ANSWER`}
           message={query.response}
@@ -178,7 +161,6 @@ export default function Conversation() {
       );
       responseView = (
         <ConversationBubble
-          ref={endMessageRef}
           className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'} `}
           key={`${index}ERROR`}
           message={query.error}
@@ -190,22 +172,42 @@ export default function Conversation() {
     return responseView;
   };
 
-  const handlePaste = (e: React.ClipboardEvent) => {
-    e.preventDefault();
-    const text = e.clipboardData.getData('text/plain');
-    document.execCommand('insertText', false, text);
+  const handleInput = () => {
+    if (inputRef.current) {
+      if (window.innerWidth < 350) inputRef.current.style.height = 'auto';
+      else inputRef.current.style.height = '64px';
+      inputRef.current.style.height = `${Math.min(
+        inputRef.current.scrollHeight,
+        96,
+      )}px`;
+    }
   };
-
+  const checkScroll = () => {
+    const el = conversationRef.current;
+    if (!el) return;
+    const isBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 10;
+    setHasScrolledToLast(isBottom);
+  };
+  useEffect(() => {
+    handleInput();
+    window.addEventListener('resize', handleInput);
+    conversationRef.current?.addEventListener('scroll', checkScroll);
+    return () => {
+      window.removeEventListener('resize', handleInput);
+      conversationRef.current?.removeEventListener('scroll', checkScroll);
+    };
+  }, []);
   return (
-    <div className="flex h-screen flex-col gap-7 pb-2">
+    <div className="flex flex-col gap-1 h-full justify-end">
       {conversationId && (
         <>
+          {' '}
           <button
             title="Share"
             onClick={() => {
               setShareModalState(true);
             }}
-            className="fixed top-4 right-20 z-30 rounded-full hover:bg-bright-gray dark:hover:bg-[#28292E]"
+            className="absolute top-4 right-20 z-20 rounded-full hover:bg-bright-gray dark:hover:bg-[#28292E]"
           >
             <img
               className="m-2 h-5 w-5 filter dark:invert"
@@ -224,9 +226,10 @@ export default function Conversation() {
         </>
       )}
       <div
+        ref={conversationRef}
         onWheel={handleUserInterruption}
         onTouchMove={handleUserInterruption}
-        className="flex h-[90%] w-full flex-1 justify-center overflow-y-auto p-4 md:h-[83vh]"
+        className="flex justify-center w-full overflow-y-auto h-screen sm:mt-12"
       >
         {queries.length > 0 && !hasScrolledToLast && (
           <button
@@ -242,13 +245,13 @@ export default function Conversation() {
           </button>
         )}
 
-        {queries.length > 0 && (
-          <div className="mt-16 w-full md:w-8/12">
+        {queries.length > 0 ? (
+          <div className="w-full md:w-8/12">
             {queries.map((query, index) => {
               return (
                 <Fragment key={index}>
                   <ConversationBubble
-                    className={'mb-1 last:mb-28 md:mb-7'}
+                    className={'first:mt-5'}
                     key={`${index}QUESTION`}
                     message={query.prompt}
                     type="QUESTION"
@@ -260,35 +263,34 @@ export default function Conversation() {
               );
             })}
           </div>
+        ) : (
+          <Hero handleQuestion={handleQuestion} />
         )}
-
-        {queries.length === 0 && <Hero handleQuestion={handleQuestion} />}
       </div>
 
-      <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 pb-1 sm:w-6/12">
-        <div className="flex h-full w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black">
-          <div
+      <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 pb-1 sm:w-[62%] h-auto">
+        <div className="flex w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black">
+          <textarea
             id="inputbox"
             ref={inputRef}
             tabIndex={1}
             placeholder={t('inputPlaceholder')}
-            contentEditable
-            onPaste={handlePaste}
-            className={`inputbox-style max-h-24 w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap rounded-full bg-white pt-5 pb-[22px] text-base leading-tight opacity-100 focus:outline-none dark:bg-raisin-black dark:text-bright-gray`}
+            className={`inputbox-style h-16 w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap rounded-full bg-white pt-5 pb-[22px] text-base leading-tight opacity-100 focus:outline-none dark:bg-raisin-black dark:text-bright-gray`}
+            onInput={handleInput}
             onKeyDown={(e) => {
               if (e.key === 'Enter' && !e.shiftKey) {
                 e.preventDefault();
                 handleQuestionSubmission();
               }
             }}
-          ></div>
+          ></textarea>
           {status === 'loading' ? (
             <img
               src={isDarkTheme ? SpinnerDark : Spinner}
               className="relative right-[38px] bottom-[24px] -mr-[30px] animate-spin cursor-pointer self-end bg-transparent"
             ></img>
           ) : (
-            <div className="mx-1 cursor-pointer rounded-full p-3 text-center hover:bg-gray-3000">
+            <div className="mx-1 cursor-pointer rounded-full p-3 text-center hover:bg-gray-3000 dark:hover:bg-dark-charcoal">
               <img
                 className="ml-[4px] h-6 w-6 text-white "
                 onClick={handleQuestionSubmission}
@@ -298,7 +300,7 @@ export default function Conversation() {
           )}
         </div>
 
-        <p className="text-gray-595959 hidden w-[100vw] self-center  bg-white bg-transparent py-2 text-center text-xs dark:bg-raisin-black dark:text-bright-gray md:inline md:w-full">
+        <p className="text-gray-595959 hidden w-[100vw] self-center bg-transparent py-2 text-center text-xs dark:text-bright-gray md:inline md:w-full">
           {t('tagline')}
         </p>
       </div>

frontend/src/conversation/ConversationBubble.tsx

@@ -1,17 +1,27 @@
 import { forwardRef, useState } from 'react';
-import Avatar from '../components/Avatar';
-import CopyButton from '../components/CopyButton';
-import remarkGfm from 'remark-gfm';
-import { FEEDBACK, MESSAGE_TYPE } from './conversationModels';
-import classes from './ConversationBubble.module.css';
-import Alert from './../assets/alert.svg';
-import Like from './../assets/like.svg?react';
-import Dislike from './../assets/dislike.svg?react';
-
 import ReactMarkdown from 'react-markdown';
+import { useSelector } from 'react-redux';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { vscDarkPlus } from 'react-syntax-highlighter/dist/cjs/styles/prism';
+import remarkGfm from 'remark-gfm';
+
+import Alert from '../assets/alert.svg';
 import DocsGPT3 from '../assets/cute_docsgpt3.svg';
+import Dislike from '../assets/dislike.svg?react';
+import Document from '../assets/document.svg';
+import Like from '../assets/like.svg?react';
+import Link from '../assets/link.svg';
+import Sources from '../assets/sources.svg';
+import Avatar from '../components/Avatar';
+import CopyButton from '../components/CopyButton';
+import Sidebar from '../components/Sidebar';
+import {
+  selectChunks,
+  selectSelectedDocs,
+} from '../preferences/preferenceSlice';
+import classes from './ConversationBubble.module.css';
+import { FEEDBACK, MESSAGE_TYPE } from './conversationModels';
+
 const DisableSourceFE = import.meta.env.VITE_DISABLE_SOURCE_FE || false;
 
 const ConversationBubble = forwardRef<
@@ -29,23 +39,25 @@ const ConversationBubble = forwardRef<
   { message, type, className, feedback, handleFeedback, sources, retryBtn },
   ref,
 ) {
-  const [openSource, setOpenSource] = useState<number | null>(null);
-
+  const chunks = useSelector(selectChunks);
+  const selectedDocs = useSelector(selectSelectedDocs);
   const [isLikeHovered, setIsLikeHovered] = useState(false);
   const [isDislikeHovered, setIsDislikeHovered] = useState(false);
   const [isLikeClicked, setIsLikeClicked] = useState(false);
   const [isDislikeClicked, setIsDislikeClicked] = useState(false);
+  const [activeTooltip, setActiveTooltip] = useState<number | null>(null);
+  const [isSidebarOpen, setIsSidebarOpen] = useState<boolean>(false);
 
   let bubble;
-
   if (type === 'QUESTION') {
     bubble = (
-      <div ref={ref} className={`flex flex-row-reverse self-end ${className}`}>
+      <div
+        ref={ref}
+        className={`flex flex-row-reverse self-end flex-wrap ${className}`}
+      >
         <Avatar className="mt-2 text-2xl" avatar="🧑‍💻"></Avatar>
-        <div className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white">
-          <ReactMarkdown className="whitespace-pre-wrap break-normal leading-normal">
-            {message}
-          </ReactMarkdown>
+        <div className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white max-w-full whitespace-pre-wrap leading-normal break-normal">
+          {message}
         </div>
       </div>
     );
@@ -55,18 +67,147 @@ const ConversationBubble = forwardRef<
         ref={ref}
         className={`flex flex-wrap self-start ${className} group flex-col  dark:text-bright-gray`}
       >
-        <div className="flex flex-wrap self-start lg:flex-nowrap">
-          <Avatar
-            className="mt-2 h-12 w-12 text-2xl"
-            avatar={
-              <img
-                src={DocsGPT3}
-                alt="DocsGPT"
-                className="h-full w-full object-cover"
+        {DisableSourceFE ||
+        type === 'ERROR' ||
+        sources?.length === 0 ||
+        sources?.some((source) => source.source === 'None') ? null : !sources &&
+          chunks !== '0' &&
+          selectedDocs ? (
+          <div className="mb-4 flex flex-col flex-wrap items-start self-start lg:flex-nowrap">
+            <div className="my-2 flex flex-row items-center justify-center gap-3">
+              <Avatar
+                className="h-[26px] w-[30px] text-xl"
+                avatar={
+                  <img
+                    src={Sources}
+                    alt="Sources"
+                    className="h-full w-full object-fill"
+                  />
+                }
               />
-            }
-          />
-
+              <p className="text-base font-semibold">Sources</p>
+            </div>
+            <div className="grid grid-cols-2 gap-2 lg:grid-cols-4">
+              {Array.from({ length: 4 }).map((_, index) => (
+                <div
+                  key={index}
+                  className="flex h-28 cursor-pointer flex-col items-start gap-1 rounded-[20px] bg-gray-1000 p-4 text-purple-30 hover:bg-[#F1F1F1] hover:text-[#6D3ECC] dark:bg-gun-metal dark:hover:bg-[#2C2E3C] dark:hover:text-[#8C67D7]"
+                >
+                  <span className="h-px w-10 animate-pulse cursor-pointer rounded-[20px] bg-[#B2B2B2] p-1"></span>
+                  <span className="h-px w-24 animate-pulse cursor-pointer rounded-[20px] bg-[#B2B2B2] p-1"></span>
+                  <span className="h-px w-16 animate-pulse cursor-pointer rounded-[20px] bg-[#B2B2B2] p-1"></span>
+                  <span className="h-px w-32 animate-pulse cursor-pointer rounded-[20px] bg-[#B2B2B2] p-1"></span>
+                  <span className="h-px w-24 animate-pulse cursor-pointer rounded-[20px] bg-[#B2B2B2] p-1"></span>
+                  <span className="h-px w-20 animate-pulse cursor-pointer rounded-[20px] bg-[#B2B2B2] p-1"></span>
+                </div>
+              ))}
+            </div>
+          </div>
+        ) : (
+          sources && (
+            <div className="mb-4 flex flex-col flex-wrap items-start self-start lg:flex-nowrap">
+              <div className="my-2 flex flex-row items-center justify-center gap-3">
+                <Avatar
+                  className="h-[26px] w-[30px] text-xl"
+                  avatar={
+                    <img
+                      src={Sources}
+                      alt="Sources"
+                      className="h-full w-full object-fill"
+                    />
+                  }
+                />
+                <p className="text-base font-semibold">Sources</p>
+              </div>
+              <div className="ml-3 mr-5 max-w-[90vw] md:max-w-[70vw] lg:max-w-[50vw]">
+                <div className="grid grid-cols-2 gap-2 lg:grid-cols-4">
+                  {sources?.slice(0, 3)?.map((source, index) => (
+                    <div key={index} className="relative">
+                      <div
+                        className="h-28 cursor-pointer rounded-[20px] bg-gray-1000 p-4 hover:bg-[#F1F1F1] dark:bg-gun-metal dark:hover:bg-[#2C2E3C]"
+                        onMouseOver={() => setActiveTooltip(index)}
+                        onMouseOut={() => setActiveTooltip(null)}
+                      >
+                        <p className="ellipsis-text h-12 break-words text-xs">
+                          {source.text}
+                        </p>
+                        <div
+                          className={`mt-[14px] flex flex-row items-center gap-[6px] underline-offset-2 ${
+                            source.source && source.source !== 'local'
+                              ? 'hover:text-[#007DFF] hover:underline dark:hover:text-[#48A0FF]'
+                              : ''
+                          }`}
+                          onClick={() =>
+                            source.source && source.source !== 'local'
+                              ? window.open(
+                                  source.source,
+                                  '_blank',
+                                  'noopener, noreferrer',
+                                )
+                              : null
+                          }
+                        >
+                          <img
+                            src={Document}
+                            alt="Document"
+                            className="h-[17px] w-[17px] object-fill"
+                          />
+                          <p
+                            className="mt-[2px] truncate text-xs"
+                            title={
+                              source.source && source.source !== 'local'
+                                ? source.source
+                                : source.title
+                            }
+                          >
+                            {source.source && source.source !== 'local'
+                              ? source.source
+                              : source.title}
+                          </p>
+                        </div>
+                      </div>
+                      {activeTooltip === index && (
+                        <div
+                          className={`absolute left-1/2 z-30 max-h-48 w-40 translate-x-[-50%] translate-y-[3px] rounded-xl bg-[#FBFBFB] p-4 text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver sm:w-56`}
+                          onMouseOver={() => setActiveTooltip(index)}
+                          onMouseOut={() => setActiveTooltip(null)}
+                        >
+                          <p className="max-h-[164px] overflow-y-auto break-words rounded-md text-sm">
+                            {source.text}
+                          </p>
+                        </div>
+                      )}
+                    </div>
+                  ))}
+                  {(sources?.length ?? 0) > 3 && (
+                    <div
+                      className="flex h-28 cursor-pointer flex-col-reverse rounded-[20px] bg-gray-1000 p-4 text-purple-30 hover:bg-[#F1F1F1] hover:text-[#6D3ECC] dark:bg-gun-metal dark:hover:bg-[#2C2E3C] dark:hover:text-[#8C67D7]"
+                      onClick={() => setIsSidebarOpen(true)}
+                    >
+                      <p className="ellipsis-text h-22 text-xs">{`View ${
+                        sources?.length ? sources.length - 3 : 0
+                      } more`}</p>
+                    </div>
+                  )}
+                </div>
+              </div>
+            </div>
+          )
+        )}
+        <div className="flex flex-col flex-wrap items-start self-start lg:flex-nowrap">
+          <div className="my-2 flex flex-row items-center justify-center gap-3">
+            <Avatar
+              className="h-[34px] w-[34px] text-2xl"
+              avatar={
+                <img
+                  src={DocsGPT3}
+                  alt="DocsGPT"
+                  className="h-full w-full object-cover"
+                />
+              }
+            />
+            <p className="text-base font-semibold">Answer</p>
+          </div>
           <div
             className={`ml-2 mr-5 flex max-w-[90vw] rounded-[28px] bg-gray-1000 py-[14px] px-7 dark:bg-gun-metal md:max-w-[70vw] lg:max-w-[50vw] ${
               type === 'ERROR'
@@ -77,7 +218,7 @@ const ConversationBubble = forwardRef<
             {type === 'ERROR' && (
               <>
                 <img src={Alert} alt="alert" className="mr-2 inline" />
-                <div className="absolute -right-32 top-1/2 -translate-y-1/2">
+                <div className="absolute right-0 lg:-right-32 top-1/2 translate-y-full lg:-translate-y-1/2">
                   {retryBtn}
                 </div>
               </>
@@ -86,15 +227,16 @@ const ConversationBubble = forwardRef<
               className="whitespace-pre-wrap break-normal leading-normal"
               remarkPlugins={[remarkGfm]}
               components={{
-                code({ node, inline, className, children, ...props }) {
+                code(props) {
+                  const { children, className, node, ref, ...rest } = props;
                   const match = /language-(\w+)/.exec(className || '');
 
-                  return !inline && match ? (
+                  return match ? (
                     <div className="group relative">
                       <SyntaxHighlighter
+                        {...rest}
                         PreTag="div"
                         language={match[1]}
-                        {...props}
                         style={vscDarkPlus}
                       >
                         {String(children).replace(/\n$/, '')}
@@ -109,7 +251,10 @@ const ConversationBubble = forwardRef<
                       </div>
                     </div>
                   ) : (
-                    <code className={className ? className : ''} {...props}>
+                    <code
+                      className={className ? className : 'whitespace-pre-line'}
+                      {...props}
+                    >
                       {children}
                     </code>
                   );
@@ -165,51 +310,9 @@ const ConversationBubble = forwardRef<
             >
               {message}
             </ReactMarkdown>
-            {DisableSourceFE ||
-            type === 'ERROR' ||
-            !sources ||
-            sources.length === 0 ? null : (
-              <>
-                <span className="mt-3 h-px w-full bg-[#DEDEDE]"></span>
-                <div className="mt-3 flex w-full flex-row flex-wrap items-center justify-start gap-2">
-                  <div className="py-1 text-base font-semibold">Sources:</div>
-                  <div className="flex flex-row flex-wrap items-center justify-start gap-2">
-                    {sources?.map((source, index) => (
-                      <div
-                        key={index}
-                        className={`max-w-xs cursor-pointer rounded-[28px] px-4 py-1 sm:max-w-sm md:max-w-md ${
-                          openSource === index
-                            ? 'bg-[#007DFF]'
-                            : 'bg-[#D7EBFD] hover:bg-[#BFE1FF]'
-                        }`}
-                        onClick={() =>
-                          source.source !== 'local'
-                            ? window.open(
-                                source.source,
-                                '_blank',
-                                'noopener, noreferrer',
-                              )
-                            : setOpenSource(openSource === index ? null : index)
-                        }
-                      >
-                        <p
-                          className={`truncate text-center text-base font-medium ${
-                            openSource === index
-                              ? 'text-white'
-                              : 'text-[#007DFF]'
-                          }`}
-                        >
-                          {index + 1}. {source.title.substring(0, 45)}
-                        </p>
-                      </div>
-                    ))}
-                  </div>
-                </div>
-              </>
-            )}
           </div>
         </div>
-        <div className="my-2 flex justify-start lg:ml-12">
+        <div className="my-2 ml-2 flex justify-start">
           <div
             className={`relative mr-5  block items-center justify-center lg:invisible 
             ${type !== 'ERROR' ? 'group-hover:lg:visible' : ''}`}
@@ -292,19 +395,15 @@ const ConversationBubble = forwardRef<
             </>
           )}
         </div>
-
-        {sources && openSource !== null && sources[openSource] && (
-          <div className="ml-10 mt-12 max-w-[300px] break-words rounded-xl bg-blue-200 p-2 dark:bg-gun-metal sm:max-w-[800px] lg:mt-2">
-            <p className="m-1 w-3/4 truncate text-xs text-gray-500 dark:text-bright-gray">
-              Source: {sources[openSource].title}
-            </p>
-
-            <div className="m-2 rounded-xl border-2 border-gray-200 bg-white p-2 dark:border-chinese-silver dark:bg-dark-charcoal">
-              <p className="text-break text-black dark:text-bright-gray">
-                {sources[openSource].text}
-              </p>
-            </div>
-          </div>
+        {sources && (
+          <Sidebar
+            isOpen={isSidebarOpen}
+            toggleState={(state: boolean) => {
+              setIsSidebarOpen(state);
+            }}
+          >
+            <AllSources sources={sources} />
+          </Sidebar>
         )}
       </div>
     );
@@ -312,4 +411,49 @@ const ConversationBubble = forwardRef<
   return bubble;
 });
 
+type AllSourcesProps = {
+  sources: { title: string; text: string; source: string }[];
+};
+
+function AllSources(sources: AllSourcesProps) {
+  return (
+    <div className="h-full w-full">
+      <div className="w-full">
+        <p className="text-left text-xl">{`${sources.sources.length} Sources`}</p>
+        <div className="mx-1 mt-2 h-[0.8px] w-full rounded-full bg-[#C4C4C4]/40 lg:w-[95%] "></div>
+      </div>
+      <div className="mt-6 flex h-[90%] w-60 flex-col items-center gap-4 overflow-y-auto sm:w-80">
+        {sources.sources.map((source, index) => (
+          <div
+            key={index}
+            className="min-h-32 w-full rounded-[20px] bg-gray-1000 p-4 dark:bg-[#28292E]"
+          >
+            <span className="flex flex-row">
+              <p
+                title={source.title}
+                className="ellipsis-text break-words text-left text-sm font-semibold"
+              >
+                {`${index + 1}. ${source.title}`}
+              </p>
+              {source.source && source.source !== 'local' ? (
+                <img
+                  src={Link}
+                  alt="Link"
+                  className="h-3 w-3 cursor-pointer object-fill"
+                  onClick={() =>
+                    window.open(source.source, '_blank', 'noopener, noreferrer')
+                  }
+                ></img>
+              ) : null}
+            </span>
+            <p className="mt-3 max-h-24 overflow-y-auto break-words rounded-md text-left text-xs text-black dark:text-chinese-silver">
+              {source.text}
+            </p>
+          </div>
+        ))}
+      </div>
+    </div>
+  );
+}
+
 export default ConversationBubble;

frontend/src/conversation/ConversationTile.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from 'react';
+import { SyntheticEvent, useEffect, useRef, useState } from 'react';
 import { useSelector } from 'react-redux';
 import Edit from '../assets/edit.svg';
 import Exit from '../assets/exit.svg';
@@ -14,6 +14,7 @@ import { selectConversationId } from '../preferences/preferenceSlice';
 import { ActiveState } from '../models/misc';
 import { ShareConversationModal } from '../modals/ShareConversationModal';
 import { useTranslation } from 'react-i18next';
+
 interface ConversationProps {
   name: string;
   id: string;
@@ -38,6 +39,7 @@ export default function ConversationTile({
   const [conversationName, setConversationsName] = useState('');
   const [isOpen, setOpen] = useState<boolean>(false);
   const [isShareModalOpen, setShareModalState] = useState<boolean>(false);
+  const [isHovered, setIsHovered] = useState(false);
   const [deleteModalState, setDeleteModalState] =
     useState<ActiveState>('INACTIVE');
   const menuRef = useRef<HTMLDivElement>(null);
@@ -46,7 +48,8 @@ export default function ConversationTile({
     setConversationsName(conversation.name);
   }, [conversation.name]);
 
-  function handleEditConversation() {
+  function handleEditConversation(event: SyntheticEvent) {
+    event.stopPropagation();
     setIsEdit(true);
     setOpen(false);
   }
@@ -77,128 +80,139 @@ export default function ConversationTile({
     setIsEdit(false);
   }
   return (
-    <div
-      ref={tileRef}
-      onClick={() => {
-        selectConversation(conversation.id);
-      }}
-      className={`my-auto mx-4 mt-4 flex h-9 cursor-pointer items-center justify-between gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
-        conversationId === conversation.id
-          ? 'bg-gray-100 dark:bg-[#28292E]'
-          : ''
-      }`}
-    >
+    <>
       <div
-        className={`flex ${
-          conversationId === conversation.id ? 'w-[75%]' : 'w-[95%]'
-        } gap-4`}
+        ref={tileRef}
+        onMouseEnter={() => {
+          setIsHovered(true);
+        }}
+        onMouseLeave={() => {
+          setIsHovered(false);
+        }}
+        onClick={() => {
+          conversationId !== conversation.id &&
+            selectConversation(conversation.id);
+        }}
+        className={`my-auto mx-4 mt-4 flex h-9 cursor-pointer items-center justify-between gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
+          conversationId === conversation.id || isOpen || isHovered
+            ? 'bg-gray-100 dark:bg-[#28292E]'
+            : ''
+        }`}
       >
-        <img
-          src={isDarkTheme ? MessageDark : Message}
-          className="ml-4 w-5 dark:text-white"
-        />
-        {isEdit ? (
-          <input
-            autoFocus
-            type="text"
-            className="h-6 w-full bg-transparent px-1 text-sm font-normal leading-6 focus:outline-[#0075FF]"
-            value={conversationName}
-            onChange={(e) => setConversationsName(e.target.value)}
+        <div className={`flex w-10/12 gap-4`}>
+          <img
+            src={isDarkTheme ? MessageDark : Message}
+            className="ml-4 w-5 dark:text-white"
           />
-        ) : (
-          <p className="my-auto overflow-hidden overflow-ellipsis whitespace-nowrap text-sm font-normal leading-6 text-eerie-black dark:text-white">
-            {conversationName}
-          </p>
-        )}
-      </div>
-      {conversationId === conversation.id && (
-        <div className="flex text-white dark:text-[#949494]" ref={menuRef}>
           {isEdit ? (
-            <div className="flex gap-1">
-              <img
-                src={CheckMark2}
-                alt="Edit"
-                className="mr-2 h-4 w-4 cursor-pointer text-white hover:opacity-50"
-                id={`img-${conversation.id}`}
-                onClick={(event) => {
-                  event.stopPropagation();
-                  handleSaveConversation({
-                    id: conversationId,
-                    name: conversationName,
-                  });
-                }}
-              />
-              <img
-                src={isEdit ? Exit : Trash}
-                alt="Exit"
-                className={`mr-4 mt-px h-3 w-3 cursor-pointer hover:opacity-50`}
-                id={`img-${conversation.id}`}
-                onClick={(event) => {
-                  event.stopPropagation();
-                  onClear();
-                }}
-              />
-            </div>
+            <input
+              autoFocus
+              type="text"
+              className="h-6 w-full bg-transparent px-1 text-sm font-normal leading-6 focus:outline-[#0075FF]"
+              value={conversationName}
+              onChange={(e) => setConversationsName(e.target.value)}
+            />
           ) : (
-            <button onClick={() => setOpen(!isOpen)}>
-              <img src={threeDots} className="mr-4 w-2" />
-            </button>
-          )}
-          {isOpen && (
-            <div className="flex-start absolute flex w-32 translate-x-1 translate-y-5 flex-col rounded-xl bg-stone-100 text-sm text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver md:w-36">
-              <button
-                onClick={() => {
-                  setShareModalState(true);
-                  setOpen(false);
-                }}
-                className="flex-start flex items-center gap-4 rounded-t-xl p-3 hover:bg-bright-gray dark:hover:bg-dark-charcoal"
-              >
-                <img
-                  src={Share}
-                  alt="Share"
-                  width={14}
-                  height={14}
-                  className="cursor-pointer hover:opacity-50"
-                  id={`img-${conversation.id}`}
-                />
-                <span>{t('convTile.share')}</span>
-              </button>
-              <button
-                onClick={(event) => {
-                  handleEditConversation();
-                }}
-                className="flex-start flex items-center gap-4 p-3 hover:bg-bright-gray dark:hover:bg-dark-charcoal"
-              >
-                <img
-                  src={Edit}
-                  alt="Edit"
-                  width={16}
-                  height={16}
-                  className="cursor-pointer hover:opacity-50"
-                  id={`img-${conversation.id}`}
-                />
-                <span>{t('convTile.rename')}</span>
-              </button>
-              <button
-                onClick={(event) => {
-                  setDeleteModalState('ACTIVE');
-                  setOpen(false);
-                }}
-                className="flex-start flex items-center gap-3 rounded-b-xl p-2 text-red-700 hover:bg-bright-gray dark:hover:bg-dark-charcoal"
-              >
-                <img
-                  src={Trash}
-                  alt="Edit"
-                  width={24}
-                  height={24}
-                  className="cursor-pointer hover:opacity-50"
-                />
-                <span>{t('convTile.delete')}</span>
-              </button>
-            </div>
+            <p className="my-auto overflow-hidden overflow-ellipsis whitespace-nowrap text-sm font-normal leading-6 text-eerie-black dark:text-white">
+              {conversationName}
+            </p>
           )}
         </div>
-      )}
+        {(conversationId === conversation.id || isHovered || isOpen) && (
+          <div className="flex text-white dark:text-[#949494]" ref={menuRef}>
+            {isEdit ? (
+              <div className="flex gap-1">
+                <img
+                  src={CheckMark2}
+                  alt="Edit"
+                  className="mr-2 h-4 w-4 cursor-pointer text-white hover:opacity-50"
+                  id={`img-${conversation.id}`}
+                  onClick={(event: SyntheticEvent) => {
+                    event.stopPropagation();
+                    handleSaveConversation({
+                      id: conversation.id,
+                      name: conversationName,
+                    });
+                  }}
+                />
+                <img
+                  src={Exit}
+                  alt="Exit"
+                  className={`mr-4 mt-px h-3 w-3 cursor-pointer filter hover:opacity-50 dark:invert`}
+                  id={`img-${conversation.id}`}
+                  onClick={(event: SyntheticEvent) => {
+                    event.stopPropagation();
+                    onClear();
+                  }}
+                />
+              </div>
+            ) : (
+              <button
+                onClick={(event: SyntheticEvent) => {
+                  event.stopPropagation();
+                  setOpen(true);
+                }}
+                className="mr-2 flex w-4 justify-center"
+              >
+                <img src={threeDots} width={8} />
+              </button>
+            )}
+            {isOpen && (
+              <div className="flex-start absolute z-30 flex w-32 translate-x-1 translate-y-5 flex-col rounded-xl bg-stone-100 text-sm text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver md:w-36">
+                <button
+                  onClick={(event: SyntheticEvent) => {
+                    event.stopPropagation();
+                    setShareModalState(true);
+                    setOpen(false);
+                  }}
+                  className="flex-start flex items-center gap-4 rounded-t-xl p-3 hover:bg-bright-gray dark:hover:bg-dark-charcoal"
+                >
+                  <img
+                    src={Share}
+                    alt="Share"
+                    width={14}
+                    height={14}
+                    className="cursor-pointer hover:opacity-50"
+                    id={`img-${conversation.id}`}
+                  />
+                  <span>{t('convTile.share')}</span>
+                </button>
+                <button
+                  onClick={handleEditConversation}
+                  className="flex-start flex items-center gap-4 p-3 hover:bg-bright-gray dark:hover:bg-dark-charcoal"
+                >
+                  <img
+                    src={Edit}
+                    alt="Edit"
+                    width={16}
+                    height={16}
+                    className="cursor-pointer hover:opacity-50"
+                    id={`img-${conversation.id}`}
+                  />
+                  <span>{t('convTile.rename')}</span>
+                </button>
+                <button
+                  onClick={(event: SyntheticEvent) => {
+                    event.stopPropagation();
+                    setDeleteModalState('ACTIVE');
+                    setOpen(false);
+                  }}
+                  className="flex-start flex items-center gap-3 rounded-b-xl p-2 text-red-700 hover:bg-bright-gray dark:hover:bg-dark-charcoal"
+                >
+                  <img
+                    src={Trash}
+                    alt="Edit"
+                    width={24}
+                    height={24}
+                    className="cursor-pointer hover:opacity-50"
+                  />
+                  <span>{t('convTile.delete')}</span>
+                </button>
+              </div>
+            )}
+          </div>
+        )}
+      </div>
       <ConfirmationModal
         message={t('convTile.deleteWarning')}
         modalState={deleteModalState}
@@ -206,14 +220,15 @@ export default function ConversationTile({
         handleSubmit={() => onDeleteConversation(conversation.id)}
         submitLabel={t('convTile.delete')}
       />
-      {isShareModalOpen && conversationId && (
+      {isShareModalOpen && (
         <ShareConversationModal
           close={() => {
             setShareModalState(false);
+            isHovered && setIsHovered(false);
           }}
-          conversationId={conversationId}
+          conversationId={conversation.id}
         />
       )}
-    </div>
+    </>
   );
 }

frontend/src/conversation/SharedConversation.tsx

@@ -1,80 +1,131 @@
-import { useState, useEffect } from 'react';
-import { useParams } from 'react-router-dom';
-import { useNavigate } from 'react-router-dom';
 import { Query } from './conversationModels';
+import { Fragment, useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
+import { useNavigate, useParams } from 'react-router-dom';
+
+import conversationService from '../api/services/conversationService';
 import ConversationBubble from './ConversationBubble';
-import { Fragment } from 'react';
-const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
-const SharedConversation = () => {
-  const params = useParams();
+import Send from '../assets/send.svg';
+import Spinner from '../assets/spinner.svg';
+import {
+  selectClientAPIKey,
+  setClientApiKey,
+  updateQuery,
+  addQuery,
+  fetchSharedAnswer,
+  selectStatus,
+} from './sharedConversationSlice';
+import { setIdentifier, setFetchedData } from './sharedConversationSlice';
+
+import { useDispatch } from 'react-redux';
+import { AppDispatch } from '../store';
+
+import {
+  selectDate,
+  selectTitle,
+  selectQueries,
+} from './sharedConversationSlice';
+import { useSelector } from 'react-redux';
+
+export const SharedConversation = () => {
   const navigate = useNavigate();
-  const { identifier } = params; //identifier is a uuid, not conversationId
-  const [queries, setQueries] = useState<Query[]>([]);
-  const [title, setTitle] = useState('');
-  const [date, setDate] = useState('');
+  const { identifier } = useParams(); //identifier is a uuid, not conversationId
+
+  const queries = useSelector(selectQueries);
+  const title = useSelector(selectTitle);
+  const date = useSelector(selectDate);
+  const apiKey = useSelector(selectClientAPIKey);
+  const status = useSelector(selectStatus);
+
+  const inputRef = useRef<HTMLDivElement>(null);
+  const sharedConversationRef = useRef<HTMLDivElement>(null);
   const { t } = useTranslation();
-  function formatISODate(isoDateStr: string) {
-    const date = new Date(isoDateStr);
+  const dispatch = useDispatch<AppDispatch>();
 
-    const monthNames = [
-      'Jan',
-      'Feb',
-      'Mar',
-      'Apr',
-      'May',
-      'June',
-      'July',
-      'Aug',
-      'Sept',
-      'Oct',
-      'Nov',
-      'Dec',
-    ];
+  const [lastQueryReturnedErr, setLastQueryReturnedErr] = useState(false);
+  const [eventInterrupt, setEventInterrupt] = useState(false);
+  const endMessageRef = useRef<HTMLDivElement>(null);
+  const handleUserInterruption = () => {
+    if (!eventInterrupt && status === 'loading') setEventInterrupt(true);
+  };
+  useEffect(() => {
+    !eventInterrupt && scrollIntoView();
+  }, [queries.length, queries[queries.length - 1]]);
 
-    const month = monthNames[date.getMonth()];
-    const day = date.getDate();
-    const year = date.getFullYear();
+  useEffect(() => {
+    identifier && dispatch(setIdentifier(identifier));
+    const element = document.getElementById('inputbox') as HTMLInputElement;
+    if (element) {
+      element.focus();
+    }
+  }, []);
 
-    let hours = date.getHours();
-    const minutes = date.getMinutes();
-    const ampm = hours >= 12 ? 'PM' : 'AM';
+  useEffect(() => {
+    if (queries.length) {
+      queries[queries.length - 1].error && setLastQueryReturnedErr(true);
+      queries[queries.length - 1].response && setLastQueryReturnedErr(false); //considering a query that initially returned error can later include a response property on retry
+    }
+  }, [queries[queries.length - 1]]);
 
-    hours = hours % 12;
-    hours = hours ? hours : 12;
-    const minutesStr = minutes < 10 ? '0' + minutes : minutes;
-    const formattedDate = `Published ${month} ${day}, ${year} at ${hours}:${minutesStr} ${ampm}`;
-    return formattedDate;
-  }
-  const fetchQueris = () => {
-    fetch(`${apiHost}/api/shared_conversation/${identifier}`)
-      .then((res) => {
-        if (res.status === 404 || res.status === 400) navigate('/pagenotfound');
-        return res.json();
-      })
-      .then((data) => {
-        if (data.success) {
-          setQueries(data.queries);
-          setTitle(data.title);
-          setDate(formatISODate(data.timestamp));
-        }
+  const scrollIntoView = () => {
+    if (!sharedConversationRef?.current || eventInterrupt) return;
+
+    if (status === 'idle' || !queries[queries.length - 1].response) {
+      sharedConversationRef.current.scrollTo({
+        behavior: 'smooth',
+        top: sharedConversationRef.current.scrollHeight,
       });
+    } else {
+      sharedConversationRef.current.scrollTop =
+        sharedConversationRef.current.scrollHeight;
+    }
   };
 
+  const fetchQueries = () => {
+    identifier &&
+      conversationService
+        .getSharedConversation(identifier || '')
+        .then((res) => {
+          if (res.status === 404 || res.status === 400)
+            navigate('/pagenotfound');
+          return res.json();
+        })
+        .then((data) => {
+          if (data.success) {
+            dispatch(
+              setFetchedData({
+                queries: data.queries,
+                title: data.title,
+                date: data.date,
+                identifier,
+              }),
+            );
+            data.api_key && dispatch(setClientApiKey(data.api_key));
+          }
+        });
+  };
+  const handlePaste = (e: React.ClipboardEvent) => {
+    e.preventDefault();
+    const text = e.clipboardData.getData('text/plain');
+    inputRef.current && (inputRef.current.innerText = text);
+  };
   const prepResponseView = (query: Query, index: number) => {
     let responseView;
     if (query.response) {
       responseView = (
         <ConversationBubble
+          ref={endMessageRef}
           className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'}`}
           key={`${index}ANSWER`}
           message={query.response}
           type={'ANSWER'}
+          sources={query.sources ?? []}
         ></ConversationBubble>
       );
     } else if (query.error) {
       responseView = (
         <ConversationBubble
+          ref={endMessageRef}
           className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'} `}
           key={`${index}ERROR`}
           message={query.error}
@@ -84,15 +135,56 @@ const SharedConversation = () => {
     }
     return responseView;
   };
+  const handleQuestionSubmission = () => {
+    if (inputRef.current?.textContent && status !== 'loading') {
+      if (lastQueryReturnedErr) {
+        // update last failed query with new prompt
+        dispatch(
+          updateQuery({
+            index: queries.length - 1,
+            query: {
+              prompt: inputRef.current.textContent,
+            },
+          }),
+        );
+        handleQuestion({
+          question: queries[queries.length - 1].prompt,
+          isRetry: true,
+        });
+      } else {
+        handleQuestion({ question: inputRef.current.textContent });
+      }
+      inputRef.current.textContent = '';
+    }
+  };
+
+  const handleQuestion = ({
+    question,
+    isRetry = false,
+  }: {
+    question: string;
+    isRetry?: boolean;
+  }) => {
+    question = question.trim();
+    if (question === '') return;
+    setEventInterrupt(false);
+    !isRetry && dispatch(addQuery({ prompt: question })); //dispatch only new queries
+    dispatch(fetchSharedAnswer({ question }));
+  };
   useEffect(() => {
-    fetchQueris();
+    fetchQueries();
   }, []);
 
   return (
     <div className="flex h-full flex-col items-center justify-between gap-2 overflow-y-hidden dark:bg-raisin-black">
-      <div className="flex w-full justify-center overflow-auto">
+      <div
+        ref={sharedConversationRef}
+        onWheel={handleUserInterruption}
+        onTouchMove={handleUserInterruption}
+        className="flex w-full justify-center overflow-auto"
+      >
         <div className="mt-0 w-11/12 md:w-10/12 lg:w-6/12">
-          <div className="mb-2 w-full border-b pb-2">
+          <div className="mb-2 w-full border-b pb-2 dark:border-b-silver">
             <h1 className="font-semi-bold text-4xl text-chinese-black dark:text-chinese-silver">
               {title}
             </h1>
@@ -111,6 +203,7 @@ const SharedConversation = () => {
               return (
                 <Fragment key={index}>
                   <ConversationBubble
+                    ref={endMessageRef}
                     className={'mb-1 last:mb-28 md:mb-7'}
                     key={`${index}QUESTION`}
                     message={query.prompt}
@@ -126,19 +219,51 @@ const SharedConversation = () => {
         </div>
       </div>
 
-      <div className=" flex flex-col items-center gap-4 pb-2">
-        <button
-          onClick={() => navigate('/')}
-          className="w-fit rounded-full bg-purple-30 p-4 text-white shadow-xl transition-colors duration-200 hover:bg-purple-taupe"
-        >
-          {t('sharedConv.button')}
-        </button>
-        <span className="hidden text-xs text-dark-charcoal dark:text-silver sm:inline">
+      <div className=" flex w-11/12 flex-col items-center gap-4 pb-2 md:w-10/12 lg:w-6/12">
+        {apiKey ? (
+          <div className="flex h-full w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black">
+            <div
+              id="inputbox"
+              ref={inputRef}
+              tabIndex={1}
+              onPaste={handlePaste}
+              placeholder={t('inputPlaceholder')}
+              contentEditable
+              className={`inputbox-style max-h-24 w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap rounded-full bg-white pt-5 pb-[22px] text-base leading-tight opacity-100 focus:outline-none dark:bg-raisin-black dark:text-bright-gray`}
+              onKeyDown={(e) => {
+                if (e.key === 'Enter' && !e.shiftKey) {
+                  e.preventDefault();
+                  handleQuestionSubmission();
+                }
+              }}
+            ></div>
+            {status === 'loading' ? (
+              <img
+                src={Spinner}
+                className="relative right-[38px] bottom-[24px] -mr-[30px] animate-spin cursor-pointer self-end bg-transparent filter dark:invert"
+              ></img>
+            ) : (
+              <div className="mx-1 cursor-pointer rounded-full p-3 text-center hover:bg-gray-3000 dark:hover:bg-dark-charcoal">
+                <img
+                  onClick={handleQuestionSubmission}
+                  className="ml-[4px] h-6 w-6 text-white filter dark:invert"
+                  src={Send}
+                ></img>
+              </div>
+            )}
+          </div>
+        ) : (
+          <button
+            onClick={() => navigate('/')}
+            className="w-fit rounded-full bg-purple-30 p-4 text-white shadow-xl transition-colors duration-200 hover:bg-purple-taupe"
+          >
+            {t('sharedConv.button')}
+          </button>
+        )}
+        <span className="mb-2 hidden text-xs text-dark-charcoal dark:text-silver sm:inline">
           {t('sharedConv.meta')}
         </span>
       </div>
     </div>
   );
 };
-
-export default SharedConversation;

frontend/src/conversation/conversationApi.ts

@@ -1,235 +0,0 @@
-import { Answer, FEEDBACK } from './conversationModels';
-import { Doc } from '../preferences/preferenceApi';
-
-const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
-
-function getDocPath(selectedDocs: Doc | null): string {
-  let docPath = 'default';
-
-  if (selectedDocs) {
-    let namePath = selectedDocs.name;
-    if (selectedDocs.language === namePath) {
-      namePath = '.project';
-    }
-    if (selectedDocs.location === 'local') {
-      docPath = 'local' + '/' + selectedDocs.name + '/';
-    } else if (selectedDocs.location === 'remote') {
-      docPath =
-        selectedDocs.language +
-        '/' +
-        namePath +
-        '/' +
-        selectedDocs.version +
-        '/' +
-        selectedDocs.model +
-        '/';
-    } else if (selectedDocs.location === 'custom') {
-      docPath = selectedDocs.docLink;
-    }
-  }
-
-  return docPath;
-}
-export function fetchAnswerApi(
-  question: string,
-  signal: AbortSignal,
-  selectedDocs: Doc | null,
-  history: Array<any> = [],
-  conversationId: string | null,
-  promptId: string | null,
-  chunks: string,
-  token_limit: number,
-): Promise<
-  | {
-      result: any;
-      answer: any;
-      sources: any;
-      conversationId: any;
-      query: string;
-    }
-  | {
-      result: any;
-      answer: any;
-      sources: any;
-      query: string;
-      conversationId: any;
-      title: any;
-    }
-> {
-  const docPath = getDocPath(selectedDocs);
-  //in history array remove all keys except prompt and response
-  history = history.map((item) => {
-    return { prompt: item.prompt, response: item.response };
-  });
-
-  return fetch(apiHost + '/api/answer', {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify({
-      question: question,
-      history: history,
-      active_docs: docPath,
-      conversation_id: conversationId,
-      prompt_id: promptId,
-      chunks: chunks,
-      token_limit: token_limit,
-    }),
-    signal,
-  })
-    .then((response) => {
-      if (response.ok) {
-        return response.json();
-      } else {
-        return Promise.reject(new Error(response.statusText));
-      }
-    })
-    .then((data) => {
-      const result = data.answer;
-      return {
-        answer: result,
-        query: question,
-        result,
-        sources: data.sources,
-        conversationId: data.conversation_id,
-      };
-    });
-}
-
-export function fetchAnswerSteaming(
-  question: string,
-  signal: AbortSignal,
-  selectedDocs: Doc | null,
-  history: Array<any> = [],
-  conversationId: string | null,
-  promptId: string | null,
-  chunks: string,
-  token_limit: number,
-  onEvent: (event: MessageEvent) => void,
-): Promise<Answer> {
-  const docPath = getDocPath(selectedDocs);
-
-  history = history.map((item) => {
-    return { prompt: item.prompt, response: item.response };
-  });
-
-  return new Promise<Answer>((resolve, reject) => {
-    const body = {
-      question: question,
-      active_docs: docPath,
-      history: JSON.stringify(history),
-      conversation_id: conversationId,
-      prompt_id: promptId,
-      chunks: chunks,
-      token_limit: token_limit,
-    };
-    fetch(apiHost + '/stream', {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify(body),
-      signal,
-    })
-      .then((response) => {
-        if (!response.body) throw Error('No response body');
-
-        const reader = response.body.getReader();
-        const decoder = new TextDecoder('utf-8');
-        let counterrr = 0;
-        const processStream = ({
-          done,
-          value,
-        }: ReadableStreamReadResult<Uint8Array>) => {
-          if (done) {
-            console.log(counterrr);
-            return;
-          }
-
-          counterrr += 1;
-
-          const chunk = decoder.decode(value);
-
-          const lines = chunk.split('\n');
-
-          for (let line of lines) {
-            if (line.trim() == '') {
-              continue;
-            }
-            if (line.startsWith('data:')) {
-              line = line.substring(5);
-            }
-
-            const messageEvent: MessageEvent = new MessageEvent('message', {
-              data: line,
-            });
-
-            onEvent(messageEvent); // handle each message
-          }
-
-          reader.read().then(processStream).catch(reject);
-        };
-
-        reader.read().then(processStream).catch(reject);
-      })
-      .catch((error) => {
-        console.error('Connection failed:', error);
-        reject(error);
-      });
-  });
-}
-export function searchEndpoint(
-  question: string,
-  selectedDocs: Doc | null,
-  conversation_id: string | null,
-  history: Array<any> = [],
-  chunks: string,
-  token_limit: number,
-) {
-  const docPath = getDocPath(selectedDocs);
-
-  const body = {
-    question: question,
-    active_docs: docPath,
-    conversation_id,
-    history,
-    chunks: chunks,
-    token_limit: token_limit,
-  };
-  return fetch(`${apiHost}/api/search`, {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify(body),
-  })
-    .then((response) => response.json())
-    .then((data) => {
-      return data;
-    })
-    .catch((err) => console.log(err));
-}
-export function sendFeedback(
-  prompt: string,
-  response: string,
-  feedback: FEEDBACK,
-) {
-  return fetch(`${apiHost}/api/feedback`, {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify({
-      question: prompt,
-      answer: response,
-      feedback: feedback,
-    }),
-  }).then((response) => {
-    if (response.ok) {
-      return Promise.resolve();
-    } else {
-      return Promise.reject();
-    }
-  });
-}

frontend/src/conversation/conversationHandlers.ts

@@ -0,0 +1,344 @@
+import conversationService from '../api/services/conversationService';
+import { Doc } from '../models/misc';
+import { Answer, FEEDBACK, RetrievalPayload } from './conversationModels';
+
+export function handleFetchAnswer(
+  question: string,
+  signal: AbortSignal,
+  selectedDocs: Doc | null,
+  history: Array<any> = [],
+  conversationId: string | null,
+  promptId: string | null,
+  chunks: string,
+  token_limit: number,
+): Promise<
+  | {
+      result: any;
+      answer: any;
+      sources: any;
+      conversationId: any;
+      query: string;
+    }
+  | {
+      result: any;
+      answer: any;
+      sources: any;
+      query: string;
+      conversationId: any;
+      title: any;
+    }
+> {
+  history = history.map((item) => {
+    return { prompt: item.prompt, response: item.response };
+  });
+  const payload: RetrievalPayload = {
+    question: question,
+    history: JSON.stringify(history),
+    conversation_id: conversationId,
+    prompt_id: promptId,
+    chunks: chunks,
+    token_limit: token_limit,
+  };
+  if (selectedDocs && 'id' in selectedDocs)
+    payload.active_docs = selectedDocs.id as string;
+  payload.retriever = selectedDocs?.retriever as string;
+  return conversationService
+    .answer(payload, signal)
+    .then((response) => {
+      if (response.ok) {
+        return response.json();
+      } else {
+        return Promise.reject(new Error(response.statusText));
+      }
+    })
+    .then((data) => {
+      const result = data.answer;
+      return {
+        answer: result,
+        query: question,
+        result,
+        sources: data.sources,
+        conversationId: data.conversation_id,
+      };
+    });
+}
+
+export function handleFetchAnswerSteaming(
+  question: string,
+  signal: AbortSignal,
+  selectedDocs: Doc | null,
+  history: Array<any> = [],
+  conversationId: string | null,
+  promptId: string | null,
+  chunks: string,
+  token_limit: number,
+  onEvent: (event: MessageEvent) => void,
+): Promise<Answer> {
+  history = history.map((item) => {
+    return { prompt: item.prompt, response: item.response };
+  });
+  const payload: RetrievalPayload = {
+    question: question,
+    history: JSON.stringify(history),
+    conversation_id: conversationId,
+    prompt_id: promptId,
+    chunks: chunks,
+    token_limit: token_limit,
+  };
+  if (selectedDocs && 'id' in selectedDocs)
+    payload.active_docs = selectedDocs.id as string;
+  payload.retriever = selectedDocs?.retriever as string;
+
+  return new Promise<Answer>((resolve, reject) => {
+    conversationService
+      .answerStream(
+        {
+          question: question,
+          active_docs: selectedDocs?.id as string,
+          history: JSON.stringify(history),
+          conversation_id: conversationId,
+          prompt_id: promptId,
+          chunks: chunks,
+          token_limit: token_limit,
+          isNoneDoc: selectedDocs === null,
+        },
+        signal,
+      )
+      .then((response) => {
+        if (!response.body) throw Error('No response body');
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder('utf-8');
+        let counterrr = 0;
+        const processStream = ({
+          done,
+          value,
+        }: ReadableStreamReadResult<Uint8Array>) => {
+          if (done) {
+            console.log(counterrr);
+            return;
+          }
+
+          counterrr += 1;
+
+          const chunk = decoder.decode(value);
+
+          const lines = chunk.split('\n');
+
+          for (let line of lines) {
+            if (line.trim() == '') {
+              continue;
+            }
+            if (line.startsWith('data:')) {
+              line = line.substring(5);
+            }
+
+            const messageEvent: MessageEvent = new MessageEvent('message', {
+              data: line,
+            });
+
+            onEvent(messageEvent); // handle each message
+          }
+
+          reader.read().then(processStream).catch(reject);
+        };
+
+        reader.read().then(processStream).catch(reject);
+      })
+      .catch((error) => {
+        console.error('Connection failed:', error);
+        reject(error);
+      });
+  });
+}
+
+export function handleSearch(
+  question: string,
+  selectedDocs: Doc | null,
+  conversation_id: string | null,
+  history: Array<any> = [],
+  chunks: string,
+  token_limit: number,
+) {
+  history = history.map((item) => {
+    return { prompt: item.prompt, response: item.response };
+  });
+  const payload: RetrievalPayload = {
+    question: question,
+    history: JSON.stringify(history),
+    conversation_id: conversation_id,
+    chunks: chunks,
+    token_limit: token_limit,
+  };
+  if (selectedDocs && 'id' in selectedDocs)
+    payload.active_docs = selectedDocs.id as string;
+  payload.retriever = selectedDocs?.retriever as string;
+  return conversationService
+    .search({
+      question: question,
+      active_docs: selectedDocs?.id as string,
+      conversation_id,
+      history,
+      chunks: chunks,
+      token_limit: token_limit,
+      isNoneDoc: selectedDocs === null,
+    })
+    .then((response) => response.json())
+    .then((data) => {
+      return data;
+    })
+    .catch((err) => console.log(err));
+}
+
+export function handleSearchViaApiKey(
+  question: string,
+  api_key: string,
+  history: Array<any> = [],
+) {
+  history = history.map((item) => {
+    return { prompt: item.prompt, response: item.response };
+  });
+  return conversationService
+    .search({
+      question: question,
+      history: JSON.stringify(history),
+      api_key: api_key,
+    })
+    .then((response) => response.json())
+    .then((data) => {
+      return data;
+    })
+    .catch((err) => console.log(err));
+}
+
+export function handleSendFeedback(
+  prompt: string,
+  response: string,
+  feedback: FEEDBACK,
+) {
+  return conversationService
+    .feedback({
+      question: prompt,
+      answer: response,
+      feedback: feedback,
+    })
+    .then((response) => {
+      if (response.ok) {
+        return Promise.resolve();
+      } else {
+        return Promise.reject();
+      }
+    });
+}
+
+export function handleFetchSharedAnswerStreaming( //for shared conversations
+  question: string,
+  signal: AbortSignal,
+  apiKey: string,
+  history: Array<any> = [],
+  onEvent: (event: MessageEvent) => void,
+): Promise<Answer> {
+  history = history.map((item) => {
+    return { prompt: item.prompt, response: item.response };
+  });
+
+  return new Promise<Answer>((resolve, reject) => {
+    const payload = {
+      question: question,
+      history: JSON.stringify(history),
+      api_key: apiKey,
+    };
+    conversationService
+      .answerStream(payload, signal)
+      .then((response) => {
+        if (!response.body) throw Error('No response body');
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder('utf-8');
+        let counterrr = 0;
+        const processStream = ({
+          done,
+          value,
+        }: ReadableStreamReadResult<Uint8Array>) => {
+          if (done) {
+            console.log(counterrr);
+            return;
+          }
+
+          counterrr += 1;
+
+          const chunk = decoder.decode(value);
+
+          const lines = chunk.split('\n');
+
+          for (let line of lines) {
+            if (line.trim() == '') {
+              continue;
+            }
+            if (line.startsWith('data:')) {
+              line = line.substring(5);
+            }
+
+            const messageEvent: MessageEvent = new MessageEvent('message', {
+              data: line,
+            });
+
+            onEvent(messageEvent); // handle each message
+          }
+
+          reader.read().then(processStream).catch(reject);
+        };
+
+        reader.read().then(processStream).catch(reject);
+      })
+      .catch((error) => {
+        console.error('Connection failed:', error);
+        reject(error);
+      });
+  });
+}
+
+export function handleFetchSharedAnswer(
+  question: string,
+  signal: AbortSignal,
+  apiKey: string,
+): Promise<
+  | {
+      result: any;
+      answer: any;
+      sources: any;
+      query: string;
+    }
+  | {
+      result: any;
+      answer: any;
+      sources: any;
+      query: string;
+      title: any;
+    }
+> {
+  return conversationService
+    .answer(
+      {
+        question: question,
+        api_key: apiKey,
+      },
+      signal,
+    )
+    .then((response) => {
+      if (response.ok) {
+        return response.json();
+      } else {
+        return Promise.reject(new Error(response.statusText));
+      }
+    })
+    .then((data) => {
+      const result = data.answer;
+      return {
+        answer: result,
+        query: question,
+        result,
+        sources: data.sources,
+      };
+    });
+}

frontend/src/conversation/conversationModels.ts

@@ -31,3 +31,13 @@ export interface Query {
   conversationId?: string | null;
   title?: string | null;
 }
+export interface RetrievalPayload {
+  question: string;
+  active_docs?: string;
+  retriever?: string;
+  history: string;
+  conversation_id: string | null;
+  prompt_id?: string | null;
+  chunks: string;
+  token_limit: number;
+}

frontend/src/conversation/conversationSlice.ts

@@ -1,10 +1,13 @@
 import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';
-import store from '../store';
-import { fetchAnswerApi, fetchAnswerSteaming } from './conversationApi';
-import { searchEndpoint } from './conversationApi';
-import { Answer, ConversationState, Query, Status } from './conversationModels';
+
 import { getConversations } from '../preferences/preferenceApi';
 import { setConversations } from '../preferences/preferenceSlice';
+import store from '../store';
+import {
+  handleFetchAnswer,
+  handleFetchAnswerSteaming,
+} from './conversationHandlers';
+import { Answer, ConversationState, Query, Status } from './conversationModels';
 
 const initialState: ConversationState = {
   queries: [],
@@ -20,7 +23,7 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
     const state = getState() as RootState;
     if (state.preference) {
       if (API_STREAMING) {
-        await fetchAnswerSteaming(
+        await handleFetchAnswerSteaming(
           question,
           signal,
           state.preference.selectedDocs!,
@@ -44,30 +47,19 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
                 .catch((error) => {
                   console.error('Failed to fetch conversations: ', error);
                 });
-
-              searchEndpoint(
-                //search for sources post streaming
-                question,
-                state.preference.selectedDocs!,
-                state.conversation.conversationId,
-                state.conversation.queries,
-                state.preference.chunks,
-                state.preference.token_limit,
-              ).then((sources) => {
-                //dispatch streaming sources
-                dispatch(
-                  updateStreamingSource({
-                    index: state.conversation.queries.length - 1,
-                    query: { sources },
-                  }),
-                );
-              });
             } else if (data.type === 'id') {
               dispatch(
                 updateConversationId({
                   query: { conversationId: data.id },
                 }),
               );
+            } else if (data.type === 'source') {
+              dispatch(
+                updateStreamingSource({
+                  index: state.conversation.queries.length - 1,
+                  query: { sources: data.source ?? [] },
+                }),
+              );
             } else if (data.type === 'error') {
               // set status to 'failed'
               dispatch(conversationSlice.actions.setStatus('failed'));
@@ -89,7 +81,7 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
           },
         );
       } else {
-        const answer = await fetchAnswerApi(
+        const answer = await handleFetchAnswer(
           question,
           signal,
           state.preference.selectedDocs!,

frontend/src/conversation/sharedConversationSlice.ts

@@ -0,0 +1,252 @@
+import { createSlice } from '@reduxjs/toolkit';
+import type { PayloadAction } from '@reduxjs/toolkit';
+import store from '../store';
+import { Query, Status, Answer } from '../conversation/conversationModels';
+import { createAsyncThunk } from '@reduxjs/toolkit';
+import {
+  handleFetchSharedAnswer,
+  handleFetchSharedAnswerStreaming,
+} from './conversationHandlers';
+
+const API_STREAMING = import.meta.env.VITE_API_STREAMING === 'true';
+interface SharedConversationsType {
+  queries: Query[];
+  apiKey?: string;
+  identifier: string;
+  status: Status;
+  date?: string;
+  title?: string;
+}
+
+const initialState: SharedConversationsType = {
+  queries: [],
+  identifier: '',
+  status: 'idle',
+};
+
+export const fetchSharedAnswer = createAsyncThunk<Answer, { question: string }>(
+  'shared/fetchAnswer',
+  async ({ question }, { dispatch, getState, signal }) => {
+    const state = getState() as RootState;
+
+    if (state.preference && state.sharedConversation.apiKey) {
+      if (API_STREAMING) {
+        await handleFetchSharedAnswerStreaming(
+          question,
+          signal,
+          state.sharedConversation.apiKey,
+          state.sharedConversation.queries,
+
+          (event) => {
+            const data = JSON.parse(event.data);
+            // check if the 'end' event has been received
+            if (data.type === 'end') {
+              // set status to 'idle'
+              dispatch(sharedConversationSlice.actions.setStatus('idle'));
+              dispatch(saveToLocalStorage());
+            } else if (data.type === 'source') {
+              dispatch(
+                updateStreamingSource({
+                  index: state.sharedConversation.queries.length - 1,
+                  query: { sources: data.source ?? [] },
+                }),
+              );
+            } else if (data.type === 'error') {
+              // set status to 'failed'
+              dispatch(sharedConversationSlice.actions.setStatus('failed'));
+              dispatch(
+                sharedConversationSlice.actions.raiseError({
+                  index: state.conversation.queries.length - 1,
+                  message: data.error,
+                }),
+              );
+            } else {
+              const result = data.answer;
+              dispatch(
+                updateStreamingQuery({
+                  index: state.sharedConversation.queries.length - 1,
+                  query: { response: result },
+                }),
+              );
+            }
+          },
+        );
+      } else {
+        const answer = await handleFetchSharedAnswer(
+          question,
+          signal,
+          state.sharedConversation.apiKey,
+        );
+        if (answer) {
+          let sourcesPrepped = [];
+          sourcesPrepped = answer.sources.map((source: { title: string }) => {
+            if (source && source.title) {
+              const titleParts = source.title.split('/');
+              return {
+                ...source,
+                title: titleParts[titleParts.length - 1],
+              };
+            }
+            return source;
+          });
+
+          dispatch(
+            updateQuery({
+              index: state.sharedConversation.queries.length - 1,
+              query: { response: answer.answer, sources: sourcesPrepped },
+            }),
+          );
+          dispatch(sharedConversationSlice.actions.setStatus('idle'));
+        }
+      }
+    }
+    return {
+      conversationId: null,
+      title: null,
+      answer: '',
+      query: question,
+      result: '',
+      sources: [],
+    };
+  },
+);
+
+export const sharedConversationSlice = createSlice({
+  name: 'sharedConversation',
+  initialState,
+  reducers: {
+    setStatus(state, action: PayloadAction<Status>) {
+      state.status = action.payload;
+    },
+    setIdentifier(state, action: PayloadAction<string>) {
+      state.identifier = action.payload;
+    },
+    setFetchedData(
+      state,
+      action: PayloadAction<{
+        queries: Query[];
+        title: string;
+        date: string;
+        identifier: string;
+      }>,
+    ) {
+      const { queries, title, identifier, date } = action.payload;
+      const previousQueriesStr = localStorage.getItem(identifier);
+      const localySavedQueries: Query[] = previousQueriesStr
+        ? JSON.parse(previousQueriesStr)
+        : [];
+      state.queries = [...queries, ...localySavedQueries];
+      state.title = title;
+      state.date = date;
+      state.identifier = identifier;
+    },
+    setClientApiKey(state, action: PayloadAction<string>) {
+      state.apiKey = action.payload;
+    },
+    addQuery(state, action: PayloadAction<Query>) {
+      state.queries.push(action.payload);
+    },
+    updateStreamingQuery(
+      state,
+      action: PayloadAction<{ index: number; query: Partial<Query> }>,
+    ) {
+      const { index, query } = action.payload;
+      if (query.response != undefined) {
+        state.queries[index].response =
+          (state.queries[index].response || '') + query.response;
+      } else {
+        state.queries[index] = {
+          ...state.queries[index],
+          ...query,
+        };
+      }
+    },
+    updateQuery(
+      state,
+      action: PayloadAction<{ index: number; query: Partial<Query> }>,
+    ) {
+      const { index, query } = action.payload;
+      state.queries[index] = {
+        ...state.queries[index],
+        ...query,
+      };
+    },
+    updateStreamingSource(
+      state,
+      action: PayloadAction<{ index: number; query: Partial<Query> }>,
+    ) {
+      const { index, query } = action.payload;
+      if (!state.queries[index].sources) {
+        state.queries[index].sources = query.sources ?? [];
+      } else if (query.sources && query.sources.length > 0) {
+        state.queries[index].sources = [
+          ...(state.queries[index].sources ?? []),
+          ...query.sources,
+        ];
+      }
+    },
+    raiseError(
+      state,
+      action: PayloadAction<{ index: number; message: string }>,
+    ) {
+      const { index, message } = action.payload;
+      state.queries[index].error = message;
+    },
+    saveToLocalStorage(state) {
+      const previousQueriesStr = localStorage.getItem(state.identifier);
+      previousQueriesStr
+        ? localStorage.setItem(
+            state.identifier,
+            JSON.stringify([
+              ...JSON.parse(previousQueriesStr),
+              state.queries[state.queries.length - 1],
+            ]),
+          )
+        : localStorage.setItem(
+            state.identifier,
+            JSON.stringify([state.queries[state.queries.length - 1]]),
+          );
+    },
+  },
+  extraReducers(builder) {
+    builder
+      .addCase(fetchSharedAnswer.pending, (state) => {
+        state.status = 'loading';
+      })
+      .addCase(fetchSharedAnswer.rejected, (state, action) => {
+        if (action.meta.aborted) {
+          state.status = 'idle';
+          return state;
+        }
+        state.status = 'failed';
+        state.queries[state.queries.length - 1].error =
+          'Something went wrong. Please check your internet connection.';
+      });
+  },
+});
+
+export const {
+  setStatus,
+  setIdentifier,
+  setFetchedData,
+  setClientApiKey,
+  updateQuery,
+  updateStreamingQuery,
+  addQuery,
+  saveToLocalStorage,
+  updateStreamingSource,
+} = sharedConversationSlice.actions;
+
+export const selectStatus = (state: RootState) =>
+  state.sharedConversation.status;
+export const selectClientAPIKey = (state: RootState) =>
+  state.sharedConversation.apiKey;
+export const selectQueries = (state: RootState) =>
+  state.sharedConversation.queries;
+export const selectTitle = (state: RootState) => state.sharedConversation.title;
+export const selectDate = (state: RootState) => state.sharedConversation.date;
+
+type RootState = ReturnType<typeof store.getState>;
+
+sharedConversationSlice;
+export default sharedConversationSlice.reducer;

frontend/src/hooks/index.ts

@@ -66,39 +66,47 @@ export function useMediaQuery() {
 }
 
 export function useDarkTheme() {
-  const [isDarkTheme, setIsDarkTheme] = useState<boolean>(
-    localStorage.getItem('selectedTheme') === 'Dark' || false,
-  );
+  const getSystemThemePreference = () => {
+    return (
+      window.matchMedia &&
+      window.matchMedia('(prefers-color-scheme: dark)').matches
+    );
+  };
+
+  const getInitialTheme = () => {
+    const storedTheme = localStorage.getItem('selectedTheme');
+    if (storedTheme === 'Dark' || storedTheme === 'Light') {
+      return storedTheme === 'Dark';
+    }
+    return getSystemThemePreference();
+  };
+
+  const [isDarkTheme, setIsDarkTheme] = useState<boolean>(getInitialTheme());
 
   useEffect(() => {
-    // Check if dark mode preference exists in local storage
-    const savedMode: string | null = localStorage.getItem('selectedTheme');
+    const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)');
+    const handleChange = () => {
+      if (localStorage.getItem('selectedTheme') === null) {
+        setIsDarkTheme(mediaQuery.matches);
+      }
+    };
 
-    // Set dark mode based on local storage preference
-    if (savedMode === 'Dark') {
-      setIsDarkTheme(true);
-      document
-        .getElementById('root')
-        ?.classList.add('dark', 'dark:bg-raisin-black');
-    } else {
-      // If no preference found, set to default (light mode)
-      setIsDarkTheme(false);
-      document.getElementById('root')?.classList.remove('dark');
-    }
+    mediaQuery.addListener(handleChange);
+    return () => mediaQuery.removeListener(handleChange);
   }, []);
+
   useEffect(() => {
     localStorage.setItem('selectedTheme', isDarkTheme ? 'Dark' : 'Light');
     if (isDarkTheme) {
-      document
-        .getElementById('root')
-        ?.classList.add('dark', 'dark:bg-raisin-black');
+      document.body?.classList.add('dark');
     } else {
-      document.getElementById('root')?.classList.remove('dark');
+      document.body?.classList.remove('dark');
     }
   }, [isDarkTheme]);
-  //method to toggle theme
-  const toggleTheme: any = () => {
+
+  const toggleTheme = () => {
     setIsDarkTheme(!isDarkTheme);
   };
-  return [isDarkTheme, toggleTheme];
+
+  return [isDarkTheme, toggleTheme] as const;
 }

frontend/src/hooks/useDefaultDocument.ts

@@ -0,0 +1,31 @@
+import React from 'react';
+import { useDispatch, useSelector } from 'react-redux';
+
+import { getDocs } from '../preferences/preferenceApi';
+import { Doc } from '../models/misc';
+import {
+  selectSelectedDocs,
+  setSelectedDocs,
+  setSourceDocs,
+} from '../preferences/preferenceSlice';
+
+export default function useDefaultDocument() {
+  const dispatch = useDispatch();
+  const selectedDoc = useSelector(selectSelectedDocs);
+
+  const fetchDocs = () => {
+    getDocs().then((data) => {
+      dispatch(setSourceDocs(data));
+      if (!selectedDoc)
+        data?.forEach((doc: Doc) => {
+          if (doc.model && doc.name === 'default') {
+            dispatch(setSelectedDocs(doc));
+          }
+        });
+    });
+  };
+
+  React.useEffect(() => {
+    fetchDocs();
+  }, []);
+}

frontend/src/index.css

@@ -2,6 +2,19 @@
 @tailwind components;
 @tailwind utilities;
 
+:root {
+  --viewport-height: 100vh;
+}
+
+@supports (height: 100dvh) {
+  :root {
+    --viewport-height: 100dvh; /* Use dvh where supported */
+  }
+}
+
+body.dark {
+  background-color: #202124; /* raisin-black */
+}
 ::-webkit-scrollbar {
   width: 8px;
 }
@@ -60,11 +73,21 @@ html {
 
 body {
   margin: 0;
-  min-height: 100vh;
+  min-height: var(--viewport-height);
   overflow-x: hidden;
   font-family: 'Inter', sans-serif;
 }
-
+/*
+Avoid over-scrolling in mobile browsers
+*/
+@media only screen and (max-width: 500px) {
+  body,
+  html {
+    min-height: var(--viewport-height);
+    position: fixed;
+    width: 100%;
+  }
+}
 /**
  * Render the `main` element consistently in IE.
  */
@@ -417,11 +440,54 @@ template {
   src: url('/fonts/Inter-Variable.ttf');
 }
 
-::-webkit-scrollbar {
-  width: 0;
+@font-face {
+  font-family: 'IBMPlexMono-Medium';
+  font-weight: 500;
+  src: url('/fonts/IBMPlexMono-Medium.ttf');
 }
 
-.inputbox-style[contenteditable] {
+::-webkit-scrollbar {
+  width: 10;
+}
+
+input:-webkit-autofill {
+  -webkit-box-shadow: 0 0 0 50px white inset;
+}
+
+input:-webkit-autofill:focus {
+  -webkit-box-shadow: 0 0 0 50px white inset;
+}
+
+@media (prefers-color-scheme: dark) {
+  input:-webkit-autofill {
+    -webkit-box-shadow: 0 0 0 50px rgb(68, 70, 84) inset;
+    -webkit-text-fill-color: white;
+  }
+
+  input:-webkit-autofill:focus {
+    -webkit-box-shadow: 0 0 0 50px rgb(68, 70, 84) inset;
+    -webkit-text-fill-color: white;
+  }
+}
+
+.inputbox-style {
+  resize: none;
   padding-left: 36px;
   padding-right: 36px;
 }
+
+.bottom-safe {
+  bottom: env(safe-area-inset-bottom, 0);
+}
+
+.ellipsis-text {
+  overflow: hidden;
+  display: -webkit-box;
+  -webkit-line-clamp: 3;
+  -webkit-box-orient: vertical;
+  text-overflow: ellipsis;
+}
+
+.logs-table {
+  font-family: 'IBMPlexMono-Medium', system-ui;
+}

frontend/src/locale/en.json

@@ -57,11 +57,17 @@
       "tokenUsage": "Token Usage"
     },
     "apiKeys": {
-      "label": "API Keys",
+      "label": "Chatbots",
       "name": "Name",
       "key": "API Key",
       "sourceDoc": "Source Document",
       "createNew": "Create New"
+    },
+    "analytics": {
+      "label": "Analytics"
+    },
+    "logs": {
+      "label": "Logs"
     }
   },
   "modals": {
@@ -71,7 +77,7 @@
       "remote": "Remote",
       "name": "Name",
       "choose": "Choose Files",
-      "info": "Please upload .pdf, .txt, .rst, .docx, .md, .zip limited to 25mb",
+      "info": "Please upload .pdf, .txt, .rst, .csv, .docx, .md, .zip limited to 25mb",
       "uploadedFiles": "Uploaded Files",
       "cancel": "Cancel",
       "train": "Train",
@@ -107,7 +113,8 @@
     "shareConv": {
       "label": "Create a public page to share",
       "note": "Source document, personal information and further conversation will remain private",
-      "create": "Create"
+      "create": "Create",
+      "option": "Allow users to prompt further"
     }
   },
   "sharedConv": {

frontend/src/locale/es.json

@@ -57,11 +57,17 @@
       "tokenUsage": "Uso de Tokens"
     },
     "apiKeys": {
-      "label": "Claves API",
+      "label": "Chatbots",
       "name": "Nombre",
       "key": "Clave de API",
       "sourceDoc": "Documento Fuente",
       "createNew": "Crear Nuevo"
+    },
+    "analytics": {
+      "label": "Analítica"
+    },
+    "logs": {
+      "label": "Registros"
     }
   },
   "modals": {
@@ -107,7 +113,8 @@
     "shareConv": {
       "label": "Crear una página pública para compartir",
       "note": "El documento original, la información personal y las conversaciones posteriores permanecerán privadas",
-      "create": "Crear"
+      "create": "Crear",
+      "option": "Permitir a los usuarios realizar más consultas."
     }
   },
   "sharedConv": {

frontend/src/locale/jp.json

@@ -57,11 +57,17 @@
       "tokenUsage": "トークン使用量"
     },
     "apiKeys": {
-      "label": "APIキー",
+      "label": "チャットボット",
       "name": "名前",
       "key": "APIキー",
       "sourceDoc": "ソースドキュメント",
       "createNew": "新規作成"
+    },
+    "analytics": {
+      "label": "分析"
+    },
+    "logs": {
+      "label": "ログ"
     }
   },
   "modals": {
@@ -107,7 +113,8 @@
     "shareConv": {
       "label": "共有ページを作成して共有する",
       "note": "ソースドキュメント、個人情報、および以降の会話は非公開のままになります",
-      "create": "作成"
+      "create": "作成",
+      "option": "ユーザーがより多くのクエリを実行できるようにします。"
     }
   },
   "sharedConv": {

frontend/src/locale/zh.json

@@ -57,11 +57,17 @@
       "tokenUsage": "令牌使用"
     },
     "apiKeys": {
-      "label": "API 密钥",
+      "label": "聊天机器人",
       "name": "名称",
       "key": "API 密钥",
       "sourceDoc": "源文档",
       "createNew": "创建新的"
+    },
+    "analytics": {
+      "label": "分析"
+    },
+    "logs": {
+      "label": "日志"
     }
   },
   "modals": {
@@ -107,7 +113,8 @@
     "shareConv": {
       "label": "创建用于分享的公共页面",
       "note": "源文档、个人信息和后续对话将保持私密",
-      "create": "创建"
+      "create": "创建",
+      "option": "允许用户进行更多查询。"
     }
   },
   "sharedConv": {

frontend/src/modals/CreateAPIKeyModal.tsx

@@ -0,0 +1,163 @@
+import React from 'react';
+import { useTranslation } from 'react-i18next';
+import { useSelector } from 'react-redux';
+
+import userService from '../api/services/userService';
+import Exit from '../assets/exit.svg';
+import Dropdown from '../components/Dropdown';
+import Input from '../components/Input';
+import { CreateAPIKeyModalProps, Doc } from '../models/misc';
+import { selectSourceDocs } from '../preferences/preferenceSlice';
+
+const embeddingsName =
+  import.meta.env.VITE_EMBEDDINGS_NAME ||
+  'huggingface_sentence-transformers/all-mpnet-base-v2';
+
+export default function CreateAPIKeyModal({
+  close,
+  createAPIKey,
+}: CreateAPIKeyModalProps) {
+  const { t } = useTranslation();
+  const docs = useSelector(selectSourceDocs);
+
+  const [APIKeyName, setAPIKeyName] = React.useState<string>('');
+  const [sourcePath, setSourcePath] = React.useState<{
+    name: string;
+    id: string;
+    type: string;
+  } | null>(null);
+  const [prompt, setPrompt] = React.useState<{
+    name: string;
+    id: string;
+    type: string;
+  } | null>(null);
+  const [activePrompts, setActivePrompts] = React.useState<
+    { name: string; id: string; type: string }[]
+  >([]);
+  const [chunk, setChunk] = React.useState<string>('2');
+  const chunkOptions = ['0', '2', '4', '6', '8', '10'];
+
+  const extractDocPaths = () =>
+    docs
+      ? docs
+          .filter((doc) => doc.model === embeddingsName)
+          .map((doc: Doc) => {
+            if ('id' in doc) {
+              return {
+                name: doc.name,
+                id: doc.id as string,
+                type: 'local',
+              };
+            }
+            return {
+              name: doc.name,
+              id: doc.id ?? 'default',
+              type: doc.type ?? 'default',
+            };
+          })
+      : [];
+
+  React.useEffect(() => {
+    const handleFetchPrompts = async () => {
+      try {
+        const response = await userService.getPrompts();
+        if (!response.ok) {
+          throw new Error('Failed to fetch prompts');
+        }
+        const promptsData = await response.json();
+        setActivePrompts(promptsData);
+      } catch (error) {
+        console.error(error);
+      }
+    };
+    handleFetchPrompts();
+  }, []);
+  return (
+    <div className="fixed top-0 left-0 z-30 flex h-screen w-screen items-center justify-center bg-gray-alpha bg-opacity-50">
+      <div className="relative w-11/12 rounded-2xl bg-white p-10 dark:bg-outer-space sm:w-[512px]">
+        <button className="absolute top-3 right-4 m-2 w-3" onClick={close}>
+          <img className="filter dark:invert" src={Exit} />
+        </button>
+        <div className="mb-6">
+          <span className="text-xl text-jet dark:text-bright-gray">
+            {t('modals.createAPIKey.label')}
+          </span>
+        </div>
+        <div className="relative mt-5 mb-4">
+          <span className="absolute left-2 -top-2 bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+            {t('modals.createAPIKey.apiKeyName')}
+          </span>
+          <Input
+            type="text"
+            className="rounded-md"
+            value={APIKeyName}
+            onChange={(e) => setAPIKeyName(e.target.value)}
+          ></Input>
+        </div>
+        <div className="my-4">
+          <Dropdown
+            placeholder={t('modals.createAPIKey.sourceDoc')}
+            selectedValue={sourcePath ? sourcePath.name : null}
+            onSelect={(selection: {
+              name: string;
+              id: string;
+              type: string;
+            }) => {
+              setSourcePath(selection);
+            }}
+            options={extractDocPaths()}
+            size="w-full"
+            rounded="xl"
+            border="border"
+          />
+        </div>
+        <div className="my-4">
+          <Dropdown
+            options={activePrompts}
+            selectedValue={prompt ? prompt.name : null}
+            placeholder={t('modals.createAPIKey.prompt')}
+            onSelect={(value: { name: string; id: string; type: string }) =>
+              setPrompt(value)
+            }
+            size="w-full"
+            border="border"
+          />
+        </div>
+        <div className="my-4">
+          <p className="mb-2 ml-2 font-semibold text-jet dark:text-bright-gray">
+            {t('modals.createAPIKey.chunks')}
+          </p>
+          <Dropdown
+            options={chunkOptions}
+            selectedValue={chunk}
+            onSelect={(value: string) => setChunk(value)}
+            size="w-full"
+            border="border"
+          />
+        </div>
+        <button
+          disabled={!sourcePath || APIKeyName.length === 0 || !prompt}
+          onClick={() => {
+            if (sourcePath && prompt) {
+              const payload: any = {
+                name: APIKeyName,
+                prompt_id: prompt.id,
+                chunks: chunk,
+              };
+              if (sourcePath.type === 'default') {
+                payload.retriever = sourcePath.id;
+              }
+              if (sourcePath.type === 'local') {
+                payload.source = sourcePath.id;
+              }
+              createAPIKey(payload);
+            }
+          }}
+          className="float-right mt-4 rounded-full bg-purple-30 px-5 py-2 text-sm text-white hover:bg-[#6F3FD1] disabled:opacity-50"
+        >
+          {t('modals.createAPIKey.create')}
+        </button>
+      </div>
+    </div>
+  );
+}