Merge branch 'main' into feature_n/lancedb

.github/workflows/ci.yml

@@ -1,10 +1,8 @@
 name: Build and push DocsGPT Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
@@ -43,5 +41,7 @@ jobs:
           context: ./application
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+          cache-to: type=inline

.github/workflows/cife.yml

@@ -1,10 +1,8 @@
 name: Build and push DocsGPT-FE Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
@@ -44,5 +42,7 @@ jobs:
           context: ./frontend
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+          cache-to: type=inline

.github/workflows/docker-develop-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v4
+        with:
+          file: './application/Dockerfile'
+          platforms: linux/amd64
+          context: ./application
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+          cache-to: type=inline

.github/workflows/docker-develop-fe-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT FE Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v4
+        with:
+          file: './frontend/Dockerfile'
+          platforms: linux/amd64
+          context: ./frontend
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+          cache-to: type=inline

.github/workflows/pytest.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.11"]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -21,7 +21,7 @@ jobs:
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
       - name: Test with pytest and generate coverage report
         run: |
-          python -m pytest --cov=application --cov=scripts --cov=extensions --cov-report=xml
+          python -m pytest --cov=application --cov-report=xml
       - name: Upload coverage reports to Codecov
         if: github.event_name == 'pull_request' && matrix.python-version == '3.11'
         uses: codecov/codecov-action@v3

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Docker Debug Frontend",
+      "request": "launch",
+      "type": "chrome",
+      "preLaunchTask": "docker-compose: debug:frontend",
+      "url": "http://127.0.0.1:5173",
+      "webRoot": "${workspaceFolder}/frontend",
+      "skipFiles": [
+        "<node_internals>/**"
+      ]
+    }
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,21 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "docker-compose",
+      "label": "docker-compose: debug:frontend",
+      "dockerCompose": {
+        "up": {
+          "detached": true,
+          "services": [
+            "frontend"
+          ],
+          "build": true
+        },
+        "files": [
+          "${workspaceFolder}/docker-compose.yaml"
+        ]
+      }
+    }
+  ]
+}

CONTRIBUTING.md

@@ -6,7 +6,7 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 
 📣 **Discussions** - Engage in conversations, start new topics, or help answer questions.
 
-🐞 **Issues** - This is where we keep track of tasks. It could be bugs,fixes or suggestions for new features.
+🐞 **Issues** - This is where we keep track of tasks. It could be bugs, fixes or suggestions for new features.
 
 🛠️ **Pull requests** - Suggest changes to our repository, either by working on existing issues or adding new features.
 
@@ -21,8 +21,9 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 - If you're interested in contributing code, here are some important things to know:
 
 - We have a frontend built on React (Vite) and a backend in Python.
-=======
-Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version that you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
+
+  
+Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
 
 ### 👨‍💻 If you're interested in contributing code, here are some important things to know:
 
@@ -43,7 +44,7 @@ Please try to follow the guidelines.
 
 ### 🖥 If you are looking to contribute to Backend (🐍 Python):
 
-- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) or [`/scripts`](https://github.com/arc53/DocsGPT/tree/main/scripts) (please disregard old [`ingest_rst.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst.py) [`ingest_rst_sphinx.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst_sphinx.py) files; they will be deprecated soon).
+- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) or [`/scripts`](https://github.com/arc53/DocsGPT/tree/main/scripts) (please disregard old [`ingest_rst.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst.py) [`ingest_rst_sphinx.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst_sphinx.py) files; these will be deprecated soon).
 - All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
 - Before submitting your Pull Request, ensure it can be queried after ingesting some test data.
   
@@ -125,4 +126,4 @@ Thank you for considering contributing to DocsGPT! 🙏
 
 ## Questions/collaboration
 Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU). We're very friendly and welcoming to new contributors, so don't hesitate to reach out.
-# Thank you so much for considering to contribute DocsGPT!🙏
+# Thank you so much for considering to contributing DocsGPT!🙏

HACKTOBERFEST.md

@@ -0,0 +1,41 @@
+# **🎉 Join the Hacktoberfest with DocsGPT and win a Free T-shirt and other prizes! 🎉**
+
+Welcome, contributors! We're excited to announce that DocsGPT is participating in Hacktoberfest. Get involved by submitting meaningful pull requests.
+
+All contributors with accepted PRs will receive a cool Holopin! 🤩 (Watch out for a reply in your PR to collect it).
+
+### 🏆 Top 50 contributors will receive a special T-shirt
+
+### 🏆 [LLM Document analysis by LexEU competition](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md): 
+A separate competition is available for those who submit new retrieval / workflow method that will analyze a Document using EU laws.
+With 200$, 100$, 50$ prize for 1st, 2nd and 3rd place respectively.
+You can find more information [here](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md)
+
+## 📜 Here's How to Contribute:
+```text
+🛠️ Code: This is the golden ticket! Make meaningful contributions through PRs.
+
+🧩 API extension: Build an app utilising DocsGPT API. We prefer submissions that showcase original ideas and turn the API into an AI agent.
+They can be a completely separate repos. 
+For example: 
+https://github.com/arc53/tg-bot-docsgpt-extenstion or 
+https://github.com/arc53/DocsGPT-cli
+
+Non-Code Contributions:
+
+📚 Wiki: Improve our documentation, create a guide or change existing documentation.
+
+🖥️ Design: Improve the UI/UX or design a new feature.
+
+📝 Blogging or Content Creation: Write articles or create videos to showcase DocsGPT or highlight your contributions!
+```
+
+### 📝 Guidelines for Pull Requests:
+- Familiarize yourself with the current contributions and our [Roadmap](https://github.com/orgs/arc53/projects/2).
+- Before contributing we highly advise that you check existing [issues](https://github.com/arc53/DocsGPT/issues) or [create](https://github.com/arc53/DocsGPT/issues/new/choose) an issue and wait to get assigned.
+- Once you are finished with your contribution, please fill in this [form](https://airtable.com/appikMaJwdHhC1SDP/pagoblCJ9W29wf6Hf/form).
+- Refer to the [Documentation](https://docs.docsgpt.cloud/).
+- Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU) server. We're here to help newcomers, so don't hesitate to jump in! Join us [here](https://discord.gg/n5BX8dh8rU).
+  
+Thank you very much for considering contributing to DocsGPT during Hacktoberfest! 🙏 Your contributions (not just simple typos) could earn you a stylish new t-shirt and other prizes as a token of our appreciation. 🎁 Join us, and let's code together! 🚀
+

README.md

@@ -23,14 +23,20 @@ Say goodbye to time-consuming manual searches, and let <strong><a href="https://
  
 </div>
 
+### 🎃 [Hacktoberfest Prizes, Rules & Q&A](https://github.com/arc53/DocsGPT/blob/main/HACKTOBERFEST.md) 🎃
+
 ### Production Support / Help for Companies:
 
 We're eager to provide personalized assistance when deploying your DocsGPT to a live environment.
 
-- [Get Enterprise / teams Demo :wave:](https://www.docsgpt.cloud/contact)
-- [Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
+<a href ="https://cal.com/arc53/docsgpt-demo-b2b">
+    <img alt="Let's chat" src="https://cal.com/book-with-cal-dark.svg" />
+</a>
 
-![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
+[Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
+
+
+<img src="https://github.com/user-attachments/assets/9a1f21de-7a15-4e42-9424-70d22ba5a913" alt="video-example-of-docs-gpt" width="1000" height="500">
 
 ## Roadmap
 

application/api/answer/routes.py

@@ -1,29 +1,38 @@
 import asyncio
+import datetime
+import json
+import logging
 import os
 import sys
-from flask import Blueprint, request, Response, current_app
-import json
-import datetime
-import logging
 import traceback
 
-from pymongo import MongoClient
+from bson.dbref import DBRef
 from bson.objectid import ObjectId
+from flask import Blueprint, current_app, make_response, request, Response
+from flask_restx import fields, Namespace, Resource
+
+from pymongo import MongoClient
 
 from application.core.settings import settings
+from application.error import bad_request
+from application.extensions import api
 from application.llm.llm_creator import LLMCreator
 from application.retriever.retriever_creator import RetrieverCreator
-from application.error import bad_request
+from application.utils import check_required_fields
 
 logger = logging.getLogger(__name__)
 
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 prompts_collection = db["prompts"]
 api_key_collection = db["api_keys"]
+user_logs_collection = db["user_logs"]
+
 answer = Blueprint("answer", __name__)
+answer_ns = Namespace("answer", description="Answer related operations", path="/")
+api.add_namespace(answer_ns)
 
 gpt_model = ""
 # to have some kind of default behaviour
@@ -31,6 +40,8 @@ if settings.LLM_NAME == "openai":
     gpt_model = "gpt-3.5-turbo"
 elif settings.LLM_NAME == "anthropic":
     gpt_model = "claude-2"
+elif settings.LLM_NAME == "groq":
+    gpt_model = "llama3-8b-8192"
 
 if settings.MODEL_NAME:  # in case there is particular model name configured
     gpt_model = settings.MODEL_NAME
@@ -74,27 +85,29 @@ def run_async_chain(chain, question, chat_history):
 
 def get_data_from_api_key(api_key):
     data = api_key_collection.find_one({"key": api_key})
-
     # # Raise custom exception if the API key is not found
     if data is None:
         raise Exception("Invalid API Key, please generate new key", 401)
+
+    if "retriever" not in data:
+        data["retriever"] = None
+
+    if "source" in data and isinstance(data["source"], DBRef):
+        source_doc = db.dereference(data["source"])
+        data["source"] = str(source_doc["_id"])
+        if "retriever" in source_doc:
+            data["retriever"] = source_doc["retriever"]
+    else:
+        data["source"] = {}
     return data
 
 
-def get_vectorstore(data):
-    if "active_docs" in data:
-        if data["active_docs"].split("/")[0] == "default":
-            vectorstore = ""
-        elif data["active_docs"].split("/")[0] == "local":
-            vectorstore = "indexes/" + data["active_docs"]
-        else:
-            vectorstore = "vectors/" + data["active_docs"]
-        if data["active_docs"] == "default":
-            vectorstore = ""
-    else:
-        vectorstore = ""
-    vectorstore = os.path.join("application", vectorstore)
-    return vectorstore
+def get_retriever(source_id: str):
+    doc = sources_collection.find_one({"_id": ObjectId(source_id)})
+    if doc is None:
+        raise Exception("Source document does not exist", 404)
+    retriever_name = None if "retriever" not in doc else doc["retriever"]
+    return retriever_name
 
 
 def is_azure_configured():
@@ -180,6 +193,13 @@ def complete_stream(
         response_full = ""
         source_log_docs = []
         answer = retriever.gen()
+        sources = retriever.search()
+        for source in sources:
+            if "text" in source:
+                source["text"] = source["text"][:100].strip() + "..."
+        if len(sources) > 0:
+            data = json.dumps({"type": "source", "source": sources})
+            yield f"data: {data}\n\n"
         for line in answer:
             if "answer" in line:
                 response_full += str(line["answer"])
@@ -203,6 +223,20 @@ def complete_stream(
             data = json.dumps({"type": "id", "id": str(conversation_id)})
             yield f"data: {data}\n\n"
 
+        retriever_params = retriever.get_params()
+        user_logs_collection.insert_one(
+            {
+                "action": "stream_answer",
+                "level": "info",
+                "user": "local",
+                "api_key": user_api_key,
+                "question": question,
+                "response": response_full,
+                "sources": source_log_docs,
+                "retriever_params": retriever_params,
+                "timestamp": datetime.datetime.now(datetime.timezone.utc),
+            }
+        )
         data = json.dumps({"type": "end"})
         yield f"data: {data}\n\n"
     except Exception as e:
@@ -218,106 +252,134 @@ def complete_stream(
         return
 
 
-@answer.route("/stream", methods=["POST"])
-def stream():
-    try:
-        data = request.get_json()
-        question = data["question"]
-        if "history" not in data:
-            history = []
-        else:
-            history = data["history"]
-            history = json.loads(history)
-        if "conversation_id" not in data:
-            conversation_id = None
-        else:
-            conversation_id = data["conversation_id"]
-        if "prompt_id" in data:
-            prompt_id = data["prompt_id"]
-        else:
-            prompt_id = "default"
-        if "selectedDocs" in data and data["selectedDocs"] is None:
-            chunks = 0
-        elif "chunks" in data:
-            chunks = int(data["chunks"])
-        else:
-            chunks = 2
-        if "token_limit" in data:
-            token_limit = data["token_limit"]
-        else:
-            token_limit = settings.DEFAULT_MAX_HISTORY
-
-        # check if active_docs or api_key is set
-
-        if "api_key" in data:
-            data_key = get_data_from_api_key(data["api_key"])
-            chunks = int(data_key["chunks"])
-            prompt_id = data_key["prompt_id"]
-            source = {"active_docs": data_key["source"]}
-            user_api_key = data["api_key"]
-        elif "active_docs" in data:
-            source = {"active_docs": data["active_docs"]}
-            user_api_key = None
-        else:
-            source = {}
-            user_api_key = None
-
-        if source["active_docs"].split("/")[0] in ["default", "local"]:
-            retriever_name = "classic"
-        else:
-            retriever_name = source["active_docs"]
-
-        current_app.logger.info(f"/stream - request_data: {data}, source: {source}",
-            extra={"data": json.dumps({"request_data": data, "source": source})}
-        )
-
-        prompt = get_prompt(prompt_id)
-
-        retriever = RetrieverCreator.create_retriever(
-            retriever_name,
-            question=question,
-            source=source,
-            chat_history=history,
-            prompt=prompt,
-            chunks=chunks,
-            token_limit=token_limit,
-            gpt_model=gpt_model,
-            user_api_key=user_api_key,
-        )
-
-        return Response(
-            complete_stream(
-                question=question,
-                retriever=retriever,
-                conversation_id=conversation_id,
-                user_api_key=user_api_key,
-                isNoneDoc=data.get("isNoneDoc"),
+@answer_ns.route("/stream")
+class Stream(Resource):
+    stream_model = api.model(
+        "StreamModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
             ),
-            mimetype="text/event-stream",
-        )
+            "history": fields.List(
+                fields.String, required=False, description="Chat history"
+            ),
+            "conversation_id": fields.String(
+                required=False, description="Conversation ID"
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "selectedDocs": fields.String(
+                required=False, description="Selected documents"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
+    )
 
-    except ValueError:
-        message = "Malformed request body"
-        print("\033[91merr", str(message), file=sys.stderr)
-        return Response(
-            error_stream_generate(message),
-            status=400,
-            mimetype="text/event-stream",
-        )
-    except Exception as e:
-        current_app.logger.error(f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
-          extra={"error": str(e), "traceback": traceback.format_exc()}
-        )
-        message = e.args[0]
-        status_code = 400
-        # # Custom exceptions with two arguments, index 1 as status code
-        if len(e.args) >= 2:
-            status_code = e.args[1]
-        return Response(
-            error_stream_generate(message),
-            status=status_code,
-            mimetype="text/event-stream",
-        )
+    @api.expect(stream_model)
+    @api.doc(description="Stream a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]
+
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            history = data.get("history", [])
+            history = json.loads(history)
+            conversation_id = data.get("conversation_id")
+            prompt_id = data.get("prompt_id", "default")
+            if "selectedDocs" in data and data["selectedDocs"] is None:
+                chunks = 0
+            else:
+                chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                prompt_id = data_key.get("prompt_id", "default")
+                source = {"active_docs": data_key.get("source")}
+                retriever_name = data_key.get("retriever", retriever_name)
+                user_api_key = data["api_key"]
+
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                retriever_name = get_retriever(data["active_docs"]) or retriever_name
+                user_api_key = None
+
+            else:
+                source = {}
+                user_api_key = None
+
+            current_app.logger.info(
+                f"/stream - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            prompt = get_prompt(prompt_id)
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=history,
+                prompt=prompt,
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            return Response(
+                complete_stream(
+                    question=question,
+                    retriever=retriever,
+                    conversation_id=conversation_id,
+                    user_api_key=user_api_key,
+                    isNoneDoc=data.get("isNoneDoc"),
+                ),
+                mimetype="text/event-stream",
+            )
+
+        except ValueError:
+            message = "Malformed request body"
+            print("\033[91merr", str(message), file=sys.stderr)
+            return Response(
+                error_stream_generate(message),
+                status=400,
+                mimetype="text/event-stream",
+            )
+        except Exception as e:
+            current_app.logger.error(
+                f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            message = e.args[0]
+            status_code = 400
+            # Custom exceptions with two arguments, index 1 as status code
+            if len(e.args) >= 2:
+                status_code = e.args[1]
+            return Response(
+                error_stream_generate(message),
+                status=status_code,
+                mimetype="text/event-stream",
+            )
 
 
 def error_stream_generate(err_response):
@@ -325,143 +387,235 @@ def error_stream_generate(err_response):
     yield f"data: {data}\n\n"
 
 
-@answer.route("/api/answer", methods=["POST"])
-def api_answer():
-    data = request.get_json()
-    question = data["question"]
-    if "history" not in data:
-        history = []
-    else:
-        history = data["history"]
-    if "conversation_id" not in data:
-        conversation_id = None
-    else:
-        conversation_id = data["conversation_id"]
-    print("-" * 5)
-    if "prompt_id" in data:
-        prompt_id = data["prompt_id"]
-    else:
-        prompt_id = "default"
-    if "chunks" in data:
-        chunks = int(data["chunks"])
-    else:
-        chunks = 2
-    if "token_limit" in data:
-        token_limit = data["token_limit"]
-    else:
-        token_limit = settings.DEFAULT_MAX_HISTORY
-
-    try:
-        # check if the vectorstore is set
-        if "api_key" in data:
-            data_key = get_data_from_api_key(data["api_key"])
-            chunks = int(data_key["chunks"])
-            prompt_id = data_key["prompt_id"]
-            source = {"active_docs": data_key["source"]}
-            user_api_key = data["api_key"]
-        else:
-            source = data
-            user_api_key = None
-
-        if source["active_docs"].split("/")[0] in ["default", "local"]:
-            retriever_name = "classic"
-        else:
-            retriever_name = source["active_docs"]
-
-        prompt = get_prompt(prompt_id)
-
-        current_app.logger.info(f"/api/answer - request_data: {data}, source: {source}",
-            extra={"data": json.dumps({"request_data": data, "source": source})}
-        )
-
-        retriever = RetrieverCreator.create_retriever(
-            retriever_name,
-            question=question,
-            source=source,
-            chat_history=history,
-            prompt=prompt,
-            chunks=chunks,
-            token_limit=token_limit,
-            gpt_model=gpt_model,
-            user_api_key=user_api_key,
-        )
-        source_log_docs = []
-        response_full = ""
-        for line in retriever.gen():
-            if "source" in line:
-                source_log_docs.append(line["source"])
-            elif "answer" in line:
-                response_full += line["answer"]
-
-        if data.get("isNoneDoc"):
-            for doc in source_log_docs:
-                doc["source"] = "None"
-
-        llm = LLMCreator.create_llm(
-            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
-        )
-
-        result = {"answer": response_full, "sources": source_log_docs}
-        result["conversation_id"] = save_conversation(
-            conversation_id, question, response_full, source_log_docs, llm
-        )
-
-        return result
-    except Exception as e:
-        current_app.logger.error(f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",
-          extra={"error": str(e), "traceback": traceback.format_exc()}
-        )
-        return bad_request(500, str(e))
-
-
-@answer.route("/api/search", methods=["POST"])
-def api_search():
-    data = request.get_json()
-    question = data["question"]
-    if "chunks" in data:
-        chunks = int(data["chunks"])
-    else:
-        chunks = 2
-    if "api_key" in data:
-        data_key = get_data_from_api_key(data["api_key"])
-        chunks = int(data_key["chunks"])
-        source = {"active_docs": data_key["source"]}
-        user_api_key = data["api_key"]
-    elif "active_docs" in data:
-        source = {"active_docs": data["active_docs"]}
-        user_api_key = None
-    else:
-        source = {}
-        user_api_key = None
-
-    if source["active_docs"].split("/")[0] in ["default", "local"]:
-        retriever_name = "classic"
-    else:
-        retriever_name = source["active_docs"]
-    if "token_limit" in data:
-        token_limit = data["token_limit"]
-    else:
-        token_limit = settings.DEFAULT_MAX_HISTORY
-        
-    current_app.logger.info(f"/api/answer - request_data: {data}, source: {source}",
-            extra={"data": json.dumps({"request_data": data, "source": source})}
+@answer_ns.route("/api/answer")
+class Answer(Resource):
+    answer_model = api.model(
+        "AnswerModel",
+        {
+            "question": fields.String(
+                required=True, description="The question to answer"
+            ),
+            "history": fields.List(
+                fields.String, required=False, description="Conversation history"
+            ),
+            "conversation_id": fields.String(
+                required=False, description="Conversation ID"
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
     )
 
-    retriever = RetrieverCreator.create_retriever(
-        retriever_name,
-        question=question,
-        source=source,
-        chat_history=[],
-        prompt="default",
-        chunks=chunks,
-        token_limit=token_limit,
-        gpt_model=gpt_model,
-        user_api_key=user_api_key,
+    @api.expect(answer_model)
+    @api.doc(description="Provide an answer based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]       
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            history = data.get("history", [])
+            conversation_id = data.get("conversation_id")
+            prompt_id = data.get("prompt_id", "default")
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                prompt_id = data_key.get("prompt_id", "default")
+                source = {"active_docs": data_key.get("source")}
+                retriever_name = data_key.get("retriever", retriever_name)
+                user_api_key = data["api_key"]
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                retriever_name = get_retriever(data["active_docs"]) or retriever_name
+                user_api_key = None
+            else:
+                source = {}
+                user_api_key = None
+
+            prompt = get_prompt(prompt_id)
+
+            current_app.logger.info(
+                f"/api/answer - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=history,
+                prompt=prompt,
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            source_log_docs = []
+            response_full = ""
+            for line in retriever.gen():
+                if "source" in line:
+                    source_log_docs.append(line["source"])
+                elif "answer" in line:
+                    response_full += line["answer"]
+
+            if data.get("isNoneDoc"):
+                for doc in source_log_docs:
+                    doc["source"] = "None"
+
+            llm = LLMCreator.create_llm(
+                settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
+            )
+
+            result = {"answer": response_full, "sources": source_log_docs}
+            result["conversation_id"] = str(
+                save_conversation(
+                    conversation_id, question, response_full, source_log_docs, llm
+                )
+            )
+            retriever_params = retriever.get_params()
+            user_logs_collection.insert_one(
+                {
+                    "action": "api_answer",
+                    "level": "info",
+                    "user": "local",
+                    "api_key": user_api_key,
+                    "question": question,
+                    "response": response_full,
+                    "sources": source_log_docs,
+                    "retriever_params": retriever_params,
+                    "timestamp": datetime.datetime.now(datetime.timezone.utc),
+                }
+            )
+
+        except Exception as e:
+            current_app.logger.error(
+                f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return bad_request(500, str(e))
+
+        return make_response(result, 200)
+
+
+@answer_ns.route("/api/search")
+class Search(Resource):
+    search_model = api.model(
+        "SearchModel",
+        {
+            "question": fields.String(
+                required=True, description="The question to search"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "api_key": fields.String(
+                required=False, description="API key for authentication"
+            ),
+            "active_docs": fields.String(
+                required=False, description="Active documents for retrieval"
+            ),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "token_limit": fields.Integer(
+                required=False, description="Limit for tokens"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
     )
-    docs = retriever.search()
 
-    if data.get("isNoneDoc"):
-        for doc in docs:
-            doc["source"] = "None"
+    @api.expect(search_model)
+    @api.doc(
+        description="Search for relevant documents based on the question and retriever"
+    )
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
 
-    return docs
+        try:
+            question = data["question"]
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                source = {"active_docs": data_key.get("source")}
+                user_api_key = data["api_key"]
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                user_api_key = None
+            else:
+                source = {}
+                user_api_key = None
+
+            current_app.logger.info(
+                f"/api/answer - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=[],
+                prompt="default",
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            docs = retriever.search()
+            retriever_params = retriever.get_params()
+
+            user_logs_collection.insert_one(
+                {
+                    "action": "api_search",
+                    "level": "info",
+                    "user": "local",
+                    "api_key": user_api_key,
+                    "question": question,
+                    "sources": docs,
+                    "retriever_params": retriever_params,
+                    "timestamp": datetime.datetime.now(datetime.timezone.utc),
+                }
+            )
+
+            if data.get("isNoneDoc"):
+                for doc in docs:
+                    doc["source"] = "None"
+
+        except Exception as e:
+            current_app.logger.error(
+                f"/api/search - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return bad_request(500, str(e))
+
+        return make_response(docs, 200)

application/api/internal/routes.py

@@ -3,18 +3,23 @@ import datetime
 from flask import Blueprint, request, send_from_directory
 from pymongo import MongoClient
 from werkzeug.utils import secure_filename
-
+from bson.objectid import ObjectId
 
 from application.core.settings import settings
+
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+
+internal = Blueprint("internal", __name__)
 
 
-internal = Blueprint('internal', __name__)
 @internal.route("/api/download", methods=["get"])
 def download_file():
     user = secure_filename(request.args.get("user"))
@@ -24,7 +29,6 @@ def download_file():
     return send_from_directory(save_dir, filename, as_attachment=True)
 
 
-
 @internal.route("/api/upload_index", methods=["POST"])
 def upload_index_files():
     """Upload two files(index.faiss, index.pkl) to the user's folder."""
@@ -35,7 +39,13 @@ def upload_index_files():
         return {"status": "no name"}
     job_name = secure_filename(request.form["name"])
     tokens = secure_filename(request.form["tokens"])
-    save_dir = os.path.join(current_dir, "indexes", user, job_name)
+    retriever = secure_filename(request.form["retriever"])
+    id = secure_filename(request.form["id"])
+    type = secure_filename(request.form["type"])
+    remote_data = request.form["remote_data"] if "remote_data" in request.form else None
+    sync_frequency = secure_filename(request.form["sync_frequency"]) if "sync_frequency" in request.form else None
+
+    save_dir = os.path.join(current_dir, "indexes", str(id))
     if settings.VECTOR_STORE == "faiss":
         if "file_faiss" not in request.files:
             print("No file part")
@@ -50,22 +60,45 @@ def upload_index_files():
         if file_pkl.filename == "":
             return {"status": "no file name"}
         # saves index files
-        
+
         if not os.path.exists(save_dir):
             os.makedirs(save_dir)
         file_faiss.save(os.path.join(save_dir, "index.faiss"))
         file_pkl.save(os.path.join(save_dir, "index.pkl"))
-    # create entry in vectors_collection
-    vectors_collection.insert_one(
-        {
-            "user": user,
-            "name": job_name,
-            "language": job_name,
-            "location": save_dir,
-            "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
-            "model": settings.EMBEDDINGS_NAME,
-            "type": "local",
-            "tokens": tokens
-        }
-    )
-    return {"status": "ok"}
+
+    existing_entry = sources_collection.find_one({"_id": ObjectId(id)})
+    if existing_entry:
+        sources_collection.update_one(
+            {"_id": ObjectId(id)},
+            {
+                "$set": {
+                    "user": user,
+                    "name": job_name,
+                    "language": job_name,
+                    "date": datetime.datetime.now(),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "type": type,
+                    "tokens": tokens,
+                    "retriever": retriever,
+                    "remote_data": remote_data,
+                    "sync_frequency": sync_frequency,
+                }
+            },
+        )
+    else:
+        sources_collection.insert_one(
+            {
+                "_id": ObjectId(id),
+                "user": user,
+                "name": job_name,
+                "language": job_name,
+                "date": datetime.datetime.now(),
+                "model": settings.EMBEDDINGS_NAME,
+                "type": type,
+                "tokens": tokens,
+                "retriever": retriever,
+                "remote_data": remote_data,
+                "sync_frequency": sync_frequency,
+            }
+        )
+    return {"status": "ok"}

application/api/user/tasks.py

@@ -1,12 +1,38 @@
-from application.worker import ingest_worker, remote_worker
+from datetime import timedelta
+
 from application.celery_init import celery
+from application.worker import ingest_worker, remote_worker, sync_worker
+
 
 @celery.task(bind=True)
 def ingest(self, directory, formats, name_job, filename, user):
     resp = ingest_worker(self, directory, formats, name_job, filename, user)
     return resp
 
+
 @celery.task(bind=True)
 def ingest_remote(self, source_data, job_name, user, loader):
     resp = remote_worker(self, source_data, job_name, user, loader)
     return resp
+
+
+@celery.task(bind=True)
+def schedule_syncs(self, frequency):
+    resp = sync_worker(self, frequency)
+    return resp
+
+
+@celery.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+    sender.add_periodic_task(
+        timedelta(days=1),
+        schedule_syncs.s("daily"),
+    )
+    sender.add_periodic_task(
+        timedelta(weeks=1),
+        schedule_syncs.s("weekly"),
+    )
+    sender.add_periodic_task(
+        timedelta(days=30),
+        schedule_syncs.s("monthly"),
+    )

application/app.py

@@ -1,15 +1,19 @@
 import platform
+
 import dotenv
-from application.celery_init import celery
-from flask import Flask, request, redirect
-from application.core.settings import settings
-from application.api.user.routes import user
+from flask import Flask, redirect, request
+
 from application.api.answer.routes import answer
 from application.api.internal.routes import internal
+from application.api.user.routes import user
+from application.celery_init import celery
 from application.core.logging_config import setup_logging
+from application.core.settings import settings
+from application.extensions import api
 
 if platform.system() == "Windows":
     import pathlib
+
     pathlib.PosixPath = pathlib.WindowsPath
 
 dotenv.load_dotenv()
@@ -23,16 +27,19 @@ app.config.update(
     UPLOAD_FOLDER="inputs",
     CELERY_BROKER_URL=settings.CELERY_BROKER_URL,
     CELERY_RESULT_BACKEND=settings.CELERY_RESULT_BACKEND,
-    MONGO_URI=settings.MONGO_URI
+    MONGO_URI=settings.MONGO_URI,
 )
 celery.config_from_object("application.celeryconfig")
+api.init_app(app)
+
 
 @app.route("/")
 def home():
-    if request.remote_addr in ('0.0.0.0', '127.0.0.1', 'localhost', '172.18.0.1'):
-        return redirect('http://localhost:5173')
+    if request.remote_addr in ("0.0.0.0", "127.0.0.1", "localhost", "172.18.0.1"):
+        return redirect("http://localhost:5173")
     else:
-        return 'Welcome to DocsGPT Backend!'
+        return "Welcome to DocsGPT Backend!"
+
 
 @app.after_request
 def after_request(response):
@@ -41,6 +48,6 @@ def after_request(response):
     response.headers.add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS")
     return response
 
+
 if __name__ == "__main__":
     app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)
-

application/cache.py

@@ -0,0 +1,93 @@
+import redis
+import time
+import json
+import logging
+from threading import Lock
+from application.core.settings import settings
+from application.utils import get_hash
+
+logger = logging.getLogger(__name__)
+
+_redis_instance = None
+_instance_lock = Lock()
+
+def get_redis_instance():
+    global _redis_instance
+    if _redis_instance is None:
+        with _instance_lock:
+            if _redis_instance is None:
+                try:
+                    _redis_instance = redis.Redis.from_url(settings.CACHE_REDIS_URL, socket_connect_timeout=2)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+                    _redis_instance = None
+    return _redis_instance
+
+def gen_cache_key(*messages, model="docgpt"):
+    if not all(isinstance(msg, dict) for msg in messages):
+        raise ValueError("All messages must be dictionaries.")
+    messages_str = json.dumps(list(messages), sort_keys=True)
+    combined = f"{model}_{messages_str}"
+    cache_key = get_hash(combined)
+    return cache_key
+
+def gen_cache(func):
+    def wrapper(self, model, messages, *args, **kwargs):
+        try:
+            cache_key = gen_cache_key(*messages)
+            redis_client = get_redis_instance()
+            if redis_client:
+                try:
+                    cached_response = redis_client.get(cache_key)
+                    if cached_response:
+                        return cached_response.decode('utf-8')
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            result = func(self, model, messages, *args, **kwargs)
+            if redis_client:
+                try:
+                    redis_client.set(cache_key, result, ex=1800)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            return result
+        except ValueError as e:
+            logger.error(e)
+            return "Error: No user message found in the conversation to generate a cache key."
+    return wrapper
+
+def stream_cache(func):
+    def wrapper(self, model, messages, stream, *args, **kwargs):
+        cache_key = gen_cache_key(*messages)
+        logger.info(f"Stream cache key: {cache_key}")
+        
+        redis_client = get_redis_instance()
+        if redis_client:
+            try:
+                cached_response = redis_client.get(cache_key)
+                if cached_response:
+                    logger.info(f"Cache hit for stream key: {cache_key}")
+                    cached_response = json.loads(cached_response.decode('utf-8'))
+                    for chunk in cached_response:
+                        yield chunk
+                        time.sleep(0.03)
+                    return
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+
+        result = func(self, model, messages, stream, *args, **kwargs)
+        stream_cache_data = []
+        
+        for chunk in result:
+            stream_cache_data.append(chunk)
+            yield chunk
+        
+        if redis_client:
+            try:
+                redis_client.set(cache_key, json.dumps(stream_cache_data), ex=1800)
+                logger.info(f"Stream cache saved for key: {cache_key}")
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+        
+    return wrapper

application/core/settings.py

@@ -21,6 +21,9 @@ class Settings(BaseSettings):
     VECTOR_STORE: str = "faiss" #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb"
     RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
 
+    # LLM Cache
+    CACHE_REDIS_URL: str = "redis://localhost:6379/2"
+
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
 
     API_KEY: Optional[str] = None  # LLM api key

application/extensions.py

@@ -0,0 +1,7 @@
+from flask_restx import Api
+
+api = Api(
+    version="1.0",
+    title="DocsGPT API",
+    description="API for DocsGPT",
+)

application/llm/base.py

@@ -1,28 +1,29 @@
 from abc import ABC, abstractmethod
 from application.usage import gen_token_usage, stream_token_usage
+from application.cache import stream_cache, gen_cache
 
 
 class BaseLLM(ABC):
     def __init__(self):
         self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
 
-    def _apply_decorator(self, method, decorator, *args, **kwargs):
-        return decorator(method, *args, **kwargs)
+    def _apply_decorator(self, method, decorators, *args, **kwargs):
+        for decorator in decorators:
+            method = decorator(method)
+        return method(self, *args, **kwargs)
 
     @abstractmethod
     def _raw_gen(self, model, messages, stream, *args, **kwargs):
         pass
 
     def gen(self, model, messages, stream=False, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen, gen_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
-        )
+        decorators = [gen_token_usage, gen_cache]
+        return self._apply_decorator(self._raw_gen, decorators=decorators, model=model, messages=messages, stream=stream, *args, **kwargs)
 
     @abstractmethod
     def _raw_gen_stream(self, model, messages, stream, *args, **kwargs):
         pass
 
     def gen_stream(self, model, messages, stream=True, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen_stream, stream_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
-        )
+        decorators = [stream_cache, stream_token_usage]
+        return self._apply_decorator(self._raw_gen_stream, decorators=decorators, model=model, messages=messages, stream=stream, *args, **kwargs)

application/llm/groq.py

@@ -0,0 +1,45 @@
+from application.llm.base import BaseLLM
+
+
+
+class GroqLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from openai import OpenAI
+
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        for line in response:
+            # import sys
+            # print(line.choices[0].delta.content, file=sys.stderr)
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content

application/llm/llm_creator.py

@@ -1,3 +1,4 @@
+from application.llm.groq import GroqLLM
 from application.llm.openai import OpenAILLM, AzureOpenAILLM
 from application.llm.sagemaker import SagemakerAPILLM
 from application.llm.huggingface import HuggingFaceLLM
@@ -17,6 +18,7 @@ class LLMCreator:
         "anthropic": AnthropicLLM,
         "docsgpt": DocsGPTAPILLM,
         "premai": PremAILLM,
+        "groq": GroqLLM
     }
 
     @classmethod

application/parser/file/bulk.py

@@ -10,13 +10,14 @@ from application.parser.file.epub_parser import EpubParser
 from application.parser.file.html_parser import HTMLParser
 from application.parser.file.markdown_parser import MarkdownParser
 from application.parser.file.rst_parser import RstParser
-from application.parser.file.tabular_parser import PandasCSVParser
+from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser
 from application.parser.schema.base import Document
 
 DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
     ".pdf": PDFParser(),
     ".docx": DocxParser(),
     ".csv": PandasCSVParser(),
+    ".xlsx":ExcelParser(),
     ".epub": EpubParser(),
     ".md": MarkdownParser(),
     ".rst": RstParser(),

application/parser/file/tabular_parser.py

@@ -113,3 +113,68 @@ class PandasCSVParser(BaseParser):
             return (self._row_joiner).join(text_list)
         else:
             return text_list
+
+
+class ExcelParser(BaseParser):
+    r"""Excel (.xlsx) parser.
+
+    Parses Excel files using Pandas `read_excel` function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_excel` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the table structure on its own.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read Excel files.")
+
+        df = pd.read_excel(file, **self._pandas_config)
+
+        text_list = df.apply(
+            lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
+        ).tolist()
+
+        if self._concat_rows:
+            return (self._row_joiner).join(text_list)
+        else:
+            return text_list

application/parser/open_ai_func.py

@@ -1,9 +1,11 @@
 import os
 
-from application.vectorstore.vector_creator import VectorCreator
-from application.core.settings import settings
 from retry import retry
 
+from application.core.settings import settings
+
+from application.vectorstore.vector_creator import VectorCreator
+
 
 # from langchain_community.embeddings import HuggingFaceEmbeddings
 # from langchain_community.embeddings import HuggingFaceInstructEmbeddings
@@ -11,12 +13,14 @@ from retry import retry
 
 
 @retry(tries=10, delay=60)
-def store_add_texts_with_retry(store, i):
+def store_add_texts_with_retry(store, i, id):
+    # add source_id to the metadata
+    i.metadata["source_id"] = str(id)
     store.add_texts([i.page_content], metadatas=[i.metadata])
     # store_pine.add_texts([i.page_content], metadatas=[i.metadata])
 
 
-def call_openai_api(docs, folder_name, task_status):
+def call_openai_api(docs, folder_name, id, task_status):
     # Function to create a vector store from the documents and save it to disk
 
     if not os.path.exists(f"{folder_name}"):
@@ -32,15 +36,16 @@ def call_openai_api(docs, folder_name, task_status):
         store = VectorCreator.create_vectorstore(
             settings.VECTOR_STORE,
             docs_init=docs_init,
-            path=f"{folder_name}",
+            source_id=f"{folder_name}",
             embeddings_key=os.getenv("EMBEDDINGS_KEY"),
         )
     else:
         store = VectorCreator.create_vectorstore(
             settings.VECTOR_STORE,
-            path=f"{folder_name}",
+            source_id=str(id),
             embeddings_key=os.getenv("EMBEDDINGS_KEY"),
         )
+        store.delete_index()
     # Uncomment for MPNet embeddings
     # model_name = "sentence-transformers/all-mpnet-base-v2"
     # hf = HuggingFaceEmbeddings(model_name=model_name)
@@ -57,7 +62,7 @@ def call_openai_api(docs, folder_name, task_status):
             task_status.update_state(
                 state="PROGRESS", meta={"current": int((c1 / s1) * 100)}
             )
-            store_add_texts_with_retry(store, i)
+            store_add_texts_with_retry(store, i, id)
         except Exception as e:
             print(e)
             print("Error on ", i)
@@ -68,5 +73,3 @@ def call_openai_api(docs, folder_name, task_status):
         c1 += 1
     if settings.VECTOR_STORE == "faiss":
         store.save_local(f"{folder_name}")
-
-

application/parser/remote/github_loader.py

@@ -0,0 +1,53 @@
+import base64
+import requests
+from typing import List
+from application.parser.remote.base import BaseRemote
+from langchain_core.documents import Document
+
+class GitHubLoader(BaseRemote):
+    def __init__(self):
+        self.access_token = None
+        self.headers = {
+            "Authorization": f"token {self.access_token}"
+        } if self.access_token else {}
+        return
+
+    def fetch_file_content(self, repo_url: str, file_path: str) -> str:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
+        response = requests.get(url, headers=self.headers)
+
+        if response.status_code == 200:
+            content = response.json()
+            if content.get("encoding") == "base64":
+                try:
+                    decoded_content = base64.b64decode(content["content"]).decode("utf-8")
+                    return f"Filename: {file_path}\n\n{decoded_content}"
+                except Exception as e:
+                    print(f"Error decoding content for {file_path}: {e}")
+                    raise
+            else:
+                return f"Filename: {file_path}\n\n{content['content']}"
+        else:
+            response.raise_for_status()
+
+    def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
+        response = requests.get(url, headers={**self.headers, "Accept": "application/vnd.github.v3.raw"})
+        contents = response.json()
+        files = []
+        for item in contents:
+            if item["type"] == "file":
+                files.append(item["path"])
+            elif item["type"] == "dir":
+                files.extend(self.fetch_repo_files(repo_url, item["path"]))
+        return files
+
+    def load_data(self, repo_url: str) -> List[Document]:
+        repo_name = repo_url.split("github.com/")[-1]
+        files = self.fetch_repo_files(repo_name)
+        documents = []
+        for file_path in files:
+            content = self.fetch_file_content(repo_name, file_path)
+            documents.append(Document(page_content=content, metadata={"title": file_path, 
+            "source": f"https://github.com/{repo_name}/blob/main/{file_path}"}))
+        return documents

application/parser/remote/remote_creator.py

@@ -2,6 +2,7 @@ from application.parser.remote.sitemap_loader import SitemapLoader
 from application.parser.remote.crawler_loader import CrawlerLoader
 from application.parser.remote.web_loader import WebLoader
 from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
+from application.parser.remote.github_loader import GitHubLoader
 
 
 class RemoteCreator:
@@ -10,6 +11,7 @@ class RemoteCreator:
         "sitemap": SitemapLoader,
         "crawler": CrawlerLoader,
         "reddit": RedditPostsLoaderRemote,
+        "github": GitHubLoader,
     }
 
     @classmethod

application/requirements.txt

@@ -1,37 +1,87 @@
-anthropic==0.34.0
+anthropic==0.34.2
 boto3==1.34.153
 beautifulsoup4==4.12.3
 celery==5.3.6
-dataclasses_json==0.6.7
+dataclasses-json==0.6.7
 docx2txt==0.8
-duckduckgo-search==6.2.6
-EbookLib==0.18
-elasticsearch==8.14.0
+duckduckgo-search==6.3.0
+ebooklib==0.18
+elastic-transport==8.15.0
+elasticsearch==8.15.1
 escodegen==1.0.11
 esprima==4.0.1
-Flask==3.0.1
+esutils==1.0.1
+Flask==3.0.3
 faiss-cpu==1.8.0.post1
+flask-restx==1.3.0
 gunicorn==23.0.0
-html2text==2020.1.16
+html2text==2024.2.26
 javalang==0.13.0
-langchain==0.2.16
-langchain-community==0.2.16
-langchain-core==0.2.38
-langchain-openai==0.1.23
-lancedb==0.13.0
-openapi3_parser==1.1.16
-pandas==2.2.2
-pydantic_settings==2.4.0
+jinja2==3.1.4
+jiter==0.5.0
+jmespath==1.0.1
+joblib==1.4.2
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-spec==0.2.4
+jsonschema-specifications==2023.7.1
+kombu==5.4.2
+langchain==0.3.0
+langchain-community==0.3.0
+langchain-core==0.3.2
+langchain-openai==0.2.0
+langchain-text-splitters==0.3.0
+langsmith==0.1.125
+lazy-object-proxy==1.10.0
+lxml==5.3.0
+markupsafe==2.1.5
+marshmallow==3.22.0
+mpmath==1.3.0
+multidict==6.1.0
+mypy-extensions==1.0.0
+networkx==3.3
+numpy==1.26.4
+openai==1.46.1
+openapi-schema-validator==0.6.2
+openapi-spec-validator==0.6.0
+openapi3-parser==1.1.18
+orjson==3.10.7
+packaging==24.1
+pandas==2.2.3
+openpyxl==3.1.5
+pathable==0.4.3
+pillow==10.4.0
+portalocker==2.10.1
+prance==23.6.21.0
+primp==0.6.3
+prompt-toolkit==3.0.47
+protobuf==5.28.2
+py==1.11.0
+pydantic==2.9.2
+pydantic-core==2.23.4
+pydantic-settings==2.4.0
 pymongo==4.8.0
-PyPDF2==3.0.1
+pypdf2==3.0.1
+python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
 qdrant-client==1.11.0
 redis==5.0.1
-Requests==2.32.0
+referencing==0.30.2
+regex==2024.9.11
+requests==2.32.3
 retry==0.9.2
-sentence-transformers
+sentence-transformers==3.0.1
 tiktoken==0.7.0
-torch
-tqdm==4.66.3
-transformers==4.44.0
-Werkzeug==3.0.3
+tokenizers==0.19.1
+torch==2.4.1
+tqdm==4.66.5
+transformers==4.44.2
+typing-extensions==4.12.2
+typing-inspect==0.9.0
+tzdata==2024.2
+urllib3==2.2.3
+vine==5.1.0
+wcwidth==0.2.13
+werkzeug==3.0.4
+yarl==1.11.1

application/retriever/base.py

@@ -12,3 +12,7 @@ class BaseRetriever(ABC):
     @abstractmethod
     def search(self, *args, **kwargs):
         pass
+
+    @abstractmethod
+    def get_params(self):
+        pass

application/retriever/brave_search.py

@@ -101,3 +101,15 @@ class BraveRetSearch(BaseRetriever):
 
     def search(self):
         return self._get_data()
+
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/classic_rag.py

@@ -1,4 +1,3 @@
-import os
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.vectorstore.vector_creator import VectorCreator
@@ -21,7 +20,7 @@ class ClassicRAG(BaseRetriever):
         user_api_key=None,
     ):
         self.question = question
-        self.vectorstore = self._get_vectorstore(source=source)
+        self.vectorstore = source['active_docs'] if 'active_docs' in source else None
         self.chat_history = chat_history
         self.prompt = prompt
         self.chunks = chunks
@@ -38,21 +37,6 @@ class ClassicRAG(BaseRetriever):
         )
         self.user_api_key = user_api_key
 
-    def _get_vectorstore(self, source):
-        if "active_docs" in source:
-            if source["active_docs"].split("/")[0] == "default":
-                vectorstore = ""
-            elif source["active_docs"].split("/")[0] == "local":
-                vectorstore = "indexes/" + source["active_docs"]
-            else:
-                vectorstore = "vectors/" + source["active_docs"]
-            if source["active_docs"] == "default":
-                vectorstore = ""
-        else:
-            vectorstore = ""
-        vectorstore = os.path.join("application", vectorstore)
-        return vectorstore
-
     def _get_data(self):
         if self.chunks == 0:
             docs = []
@@ -61,13 +45,12 @@ class ClassicRAG(BaseRetriever):
                 settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
             )
             docs_temp = docsearch.search(self.question, k=self.chunks)
+            print(docs_temp)
             docs = [
                 {
-                    "title": (
-                        i.metadata["title"].split("/")[-1]
-                        if i.metadata
-                        else i.page_content
-                    ),
+                    "title": i.metadata.get(
+                        "title", i.metadata.get("post_title", i.page_content)
+                    ).split("/")[-1],
                     "text": i.page_content,
                     "source": (
                         i.metadata.get("source")
@@ -121,3 +104,15 @@ class ClassicRAG(BaseRetriever):
 
     def search(self):
         return self._get_data()
+    
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.vectorstore,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/duckduck_search.py

@@ -118,3 +118,15 @@ class DuckDuckSearch(BaseRetriever):
 
     def search(self):
         return self._get_data()
+    
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/retriever_creator.py

@@ -5,15 +5,16 @@ from application.retriever.brave_search import BraveRetSearch
 
 
 class RetrieverCreator:
-    retievers = {
+    retrievers = {
         'classic': ClassicRAG,
         'duckduck_search': DuckDuckSearch,
-        'brave_search': BraveRetSearch
+        'brave_search': BraveRetSearch,
+        'default': ClassicRAG
     }
 
     @classmethod
     def create_retriever(cls, type, *args, **kwargs):
-        retiever_class = cls.retievers.get(type.lower())
+        retiever_class = cls.retrievers.get(type.lower())
         if not retiever_class:
             raise ValueError(f"No retievers class found for type {type}")
         return retiever_class(*args, **kwargs)

application/utils.py

@@ -1,22 +1,48 @@
 import tiktoken
+import hashlib
+from flask import jsonify, make_response
+
 
 _encoding = None
 
+
 def get_encoding():
     global _encoding
     if _encoding is None:
         _encoding = tiktoken.get_encoding("cl100k_base")
     return _encoding
 
+
 def num_tokens_from_string(string: str) -> int:
     encoding = get_encoding()
     num_tokens = len(encoding.encode(string))
     return num_tokens
 
+
 def count_tokens_docs(docs):
     docs_content = ""
     for doc in docs:
         docs_content += doc.page_content
 
     tokens = num_tokens_from_string(docs_content)
-    return tokens
+    return tokens
+
+
+def check_required_fields(data, required_fields):
+    missing_fields = [field for field in required_fields if field not in data]
+    if missing_fields:
+        return make_response(
+            jsonify(
+                {
+                    "success": False,
+                    "message": f"Missing fields: {', '.join(missing_fields)}",
+                }
+            ),
+            400,
+        )
+    return None
+
+
+def get_hash(data):
+    return hashlib.md5(data.encode()).hexdigest()
+

application/vectorstore/elasticsearch.py

@@ -9,9 +9,9 @@ import elasticsearch
 class ElasticsearchStore(BaseVectorStore):
     _es_connection = None  # Class attribute to hold the Elasticsearch connection
 
-    def __init__(self, path, embeddings_key, index_name=settings.ELASTIC_INDEX):
+    def __init__(self, source_id, embeddings_key, index_name=settings.ELASTIC_INDEX):
         super().__init__()
-        self.path = path.replace("application/indexes/", "").rstrip("/")
+        self.source_id = source_id.replace("application/indexes/", "").rstrip("/")
         self.embeddings_key = embeddings_key
         self.index_name = index_name
         
@@ -81,7 +81,7 @@ class ElasticsearchStore(BaseVectorStore):
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
         vector = embeddings.embed_query(question)
         knn = {
-            "filter": [{"match": {"metadata.store.keyword": self.path}}],
+            "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
             "field": "vector",
             "k": k,
             "num_candidates": 100,
@@ -100,7 +100,7 @@ class ElasticsearchStore(BaseVectorStore):
                             }
                         }
                     ],
-                    "filter": [{"match": {"metadata.store.keyword": self.path}}],
+                    "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
                 }
             },
             "rank": {"rrf": {}},
@@ -209,5 +209,4 @@ class ElasticsearchStore(BaseVectorStore):
 
     def delete_index(self):
         self._es_connection.delete_by_query(index=self.index_name, query={"match": {
-                                      "metadata.store.keyword": self.path}},)
-
+                                      "metadata.source_id.keyword": self.source_id}},)

application/vectorstore/faiss.py

@@ -1,22 +1,29 @@
 from langchain_community.vectorstores import FAISS
 from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
+import os
+
+def get_vectorstore(path: str) -> str:
+    if path:
+        vectorstore = os.path.join("application", "indexes", path)
+    else:
+        vectorstore = os.path.join("application")
+    return vectorstore
 
 class FaissStore(BaseVectorStore):
-
-    def __init__(self, path, embeddings_key, docs_init=None):
+    def __init__(self, source_id: str, embeddings_key: str, docs_init=None):
         super().__init__()
-        self.path = path
+        self.path = get_vectorstore(source_id)
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
-        if docs_init:
-            self.docsearch = FAISS.from_documents(
-                docs_init, embeddings
-            )
-        else:
-            self.docsearch = FAISS.load_local(
-                self.path, embeddings, 
-                allow_dangerous_deserialization=True
-            )
+
+        try:
+            if docs_init:
+                self.docsearch = FAISS.from_documents(docs_init, embeddings)
+            else:
+                self.docsearch = FAISS.load_local(self.path, embeddings, allow_dangerous_deserialization=True)
+        except Exception:
+            raise
+
         self.assert_embedding_dimensions(embeddings)
 
     def search(self, *args, **kwargs):
@@ -32,16 +39,12 @@ class FaissStore(BaseVectorStore):
         return self.docsearch.delete(*args, **kwargs)
 
     def assert_embedding_dimensions(self, embeddings):
-        """
-        Check that the word embedding dimension of the docsearch index matches
-        the dimension of the word embeddings used 
-        """
+        """Check that the word embedding dimension of the docsearch index matches the dimension of the word embeddings used."""
         if settings.EMBEDDINGS_NAME == "huggingface_sentence-transformers/all-mpnet-base-v2":
-            try:
-                word_embedding_dimension = embeddings.dimension
-            except AttributeError as e:
-                raise AttributeError("'dimension' attribute not found in embeddings instance. Make sure the embeddings object is properly initialized.") from e
+            word_embedding_dimension = getattr(embeddings, 'dimension', None)
+            if word_embedding_dimension is None:
+                raise AttributeError("'dimension' attribute not found in embeddings instance.")
+            
             docsearch_index_dimension = self.docsearch.index.d
             if word_embedding_dimension != docsearch_index_dimension:
-                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) " +
-                                 f"!= docsearch index dimension ({docsearch_index_dimension})")
+                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) != docsearch index dimension ({docsearch_index_dimension})")

application/vectorstore/mongodb.py

@@ -1,11 +1,12 @@
-from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
 from application.vectorstore.document_class import Document
 
+
 class MongoDBVectorStore(BaseVectorStore):
     def __init__(
         self,
-        path: str = "",
+        source_id: str = "",
         embeddings_key: str = "embeddings",
         collection: str = "documents",
         index_name: str = "vector_search_index",
@@ -18,7 +19,7 @@ class MongoDBVectorStore(BaseVectorStore):
         self._embedding_key = embedding_key
         self._embeddings_key = embeddings_key
         self._mongo_uri = settings.MONGO_URI
-        self._path = path.replace("application/indexes/", "").rstrip("/")
+        self._source_id = source_id.replace("application/indexes/", "").rstrip("/")
         self._embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
 
         try:
@@ -33,27 +34,24 @@ class MongoDBVectorStore(BaseVectorStore):
         self._database = self._client[database]
         self._collection = self._database[collection]
 
-        
     def search(self, question, k=2, *args, **kwargs):
         query_vector = self._embedding.embed_query(question)
 
         pipeline = [
             {
                 "$vectorSearch": {
-                    "queryVector": query_vector, 
+                    "queryVector": query_vector,
                     "path": self._embedding_key,
-                    "limit": k, 
-                    "numCandidates": k * 10, 
+                    "limit": k,
+                    "numCandidates": k * 10,
                     "index": self._index_name,
-                    "filter": {
-                        "store": {"$eq": self._path}
-                    }
+                    "filter": {"source_id": {"$eq": self._source_id}},
                 }
             }
         ]
 
         cursor = self._collection.aggregate(pipeline)
-        
+
         results = []
         for doc in cursor:
             text = doc[self._text_key]
@@ -63,30 +61,32 @@ class MongoDBVectorStore(BaseVectorStore):
             metadata = doc
             results.append(Document(text, metadata))
         return results
-    
+
     def _insert_texts(self, texts, metadatas):
         if not texts:
             return []
         embeddings = self._embedding.embed_documents(texts)
+
         to_insert = [
             {self._text_key: t, self._embedding_key: embedding, **m}
             for t, m, embedding in zip(texts, metadatas, embeddings)
         ]
-        # insert the documents in MongoDB Atlas
+
         insert_result = self._collection.insert_many(to_insert)
         return insert_result.inserted_ids
-    
-    def add_texts(self,
+
+    def add_texts(
+        self,
         texts,
-        metadatas = None,
-        ids = None,
-        refresh_indices = True,
-        create_index_if_not_exists = True,
-        bulk_kwargs = None,
-        **kwargs,):
+        metadatas=None,
+        ids=None,
+        refresh_indices=True,
+        create_index_if_not_exists=True,
+        bulk_kwargs=None,
+        **kwargs,
+    ):
 
-
-        #dims = self._embedding.client[1].word_embedding_dimension
+        # dims = self._embedding.client[1].word_embedding_dimension
         # # check if index exists
         # if create_index_if_not_exists:
         #     # check if index exists
@@ -121,6 +121,6 @@ class MongoDBVectorStore(BaseVectorStore):
         if texts_batch:
             result_ids.extend(self._insert_texts(texts_batch, metadatas_batch))
         return result_ids
-    
+
     def delete_index(self, *args, **kwargs):
-        self._collection.delete_many({"store": self._path})
+        self._collection.delete_many({"source_id": self._source_id})

application/vectorstore/qdrant.py

@@ -5,12 +5,12 @@ from qdrant_client import models
 
 
 class QdrantStore(BaseVectorStore):
-    def __init__(self, path: str = "", embeddings_key: str = "embeddings"):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
         self._filter = models.Filter(
             must=[
                 models.FieldCondition(
-                    key="metadata.store",
-                    match=models.MatchValue(value=path.replace("application/indexes/", "").rstrip("/")),
+                    key="metadata.source_id",
+                    match=models.MatchValue(value=source_id.replace("application/indexes/", "").rstrip("/")),
                 )
             ]
         )

application/worker.py

@@ -1,37 +1,44 @@
+import logging
 import os
 import shutil
 import string
 import zipfile
+from collections import Counter
 from urllib.parse import urljoin
-import logging
 
 import requests
+from bson.objectid import ObjectId
+from pymongo import MongoClient
 
 from application.core.settings import settings
 from application.parser.file.bulk import SimpleDirectoryReader
-from application.parser.remote.remote_creator import RemoteCreator
 from application.parser.open_ai_func import call_openai_api
+from application.parser.remote.remote_creator import RemoteCreator
 from application.parser.schema.base import Document
 from application.parser.token_func import group_split
 from application.utils import count_tokens_docs
 
+mongo = MongoClient(settings.MONGO_URI)
+db = mongo["docsgpt"]
+sources_collection = db["sources"]
+
+# Constants
+MIN_TOKENS = 150
+MAX_TOKENS = 1250
+RECURSION_DEPTH = 2
 
 # Define a function to extract metadata from a given filename.
 def metadata_from_filename(title):
-    store = "/".join(title.split("/")[1:3])
-    return {"title": title, "store": store}
-
+    return {"title": title}
 
 # Define a function to generate a random string of a given length.
 def generate_random_string(length):
     return "".join([string.ascii_letters[i % 52] for i in range(length)])
 
-
 current_dir = os.path.dirname(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 )
 
-
 def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
     """
     Recursively extract zip files with a limit on recursion depth.
@@ -46,9 +53,13 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
         logging.warning(f"Reached maximum recursion depth of {max_depth}")
         return
 
-    with zipfile.ZipFile(zip_path, "r") as zip_ref:
-        zip_ref.extractall(extract_to)
-    os.remove(zip_path)  # Remove the zip file after extracting
+    try:
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(extract_to)
+        os.remove(zip_path)  # Remove the zip file after extracting
+    except Exception as e:
+        logging.error(f"Error extracting zip file {zip_path}: {e}")
+        return
 
     # Check for nested zip files and extract them
     for root, dirs, files in os.walk(extract_to):
@@ -58,9 +69,43 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
                 file_path = os.path.join(root, file)
                 extract_zip_recursive(file_path, root, current_depth + 1, max_depth)
 
+def download_file(url, params, dest_path):
+    try:
+        response = requests.get(url, params=params)
+        response.raise_for_status()
+        with open(dest_path, "wb") as f:
+            f.write(response.content)
+    except requests.RequestException as e:
+        logging.error(f"Error downloading file: {e}")
+        raise
+
+def upload_index(full_path, file_data):
+    try:
+        if settings.VECTOR_STORE == "faiss":
+            files = {
+                "file_faiss": open(full_path + "/index.faiss", "rb"),
+                "file_pkl": open(full_path + "/index.pkl", "rb"),
+            }
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
+            )
+        else:
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), data=file_data
+            )
+        response.raise_for_status()
+    except requests.RequestException as e:
+        logging.error(f"Error uploading index: {e}")
+        raise
+    finally:
+        if settings.VECTOR_STORE == "faiss":
+            for file in files.values():
+                file.close()
 
 # Define the main function for ingesting and processing documents.
-def ingest_worker(self, directory, formats, name_job, filename, user):
+def ingest_worker(
+    self, directory, formats, name_job, filename, user, retriever="classic"
+):
     """
     Ingest and process documents.
 
@@ -71,43 +116,30 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         name_job (str): Name of the job for this ingestion task.
         filename (str): Name of the file to be ingested.
         user (str): Identifier for the user initiating the ingestion.
+        retriever (str): Type of retriever to use for processing the documents.
 
     Returns:
         dict: Information about the completed ingestion task, including input parameters and a "limited" flag.
     """
-    # directory = 'inputs' or 'temp'
-    # formats = [".rst", ".md"]
     input_files = None
     recursive = True
     limit = None
     exclude = True
-    # name_job = 'job1'
-    # filename = 'install.rst'
-    # user = 'local'
     sample = False
     token_check = True
-    min_tokens = 150
-    max_tokens = 1250
-    recursion_depth = 2
     full_path = os.path.join(directory, user, name_job)
 
     logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job})
-    # check if API_URL env variable is set
     file_data = {"name": name_job, "file": filename, "user": user}
-    response = requests.get(
-        urljoin(settings.API_URL, "/api/download"), params=file_data
-    )
-    file = response.content
 
     if not os.path.exists(full_path):
         os.makedirs(full_path)
-    with open(os.path.join(full_path, filename), "wb") as f:
-        f.write(file)
+    download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename))
 
     # check if file is .zip and extract it
     if filename.endswith(".zip"):
         extract_zip_recursive(
-            os.path.join(full_path, filename), full_path, 0, recursion_depth
+            os.path.join(full_path, filename), full_path, 0, RECURSION_DEPTH
         )
 
     self.update_state(state="PROGRESS", meta={"current": 1})
@@ -123,14 +155,15 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
     ).load_data()
     raw_docs = group_split(
         documents=raw_docs,
-        min_tokens=min_tokens,
-        max_tokens=max_tokens,
+        min_tokens=MIN_TOKENS,
+        max_tokens=MAX_TOKENS,
         token_check=token_check,
     )
 
     docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+    id = ObjectId()
 
-    call_openai_api(docs, full_path, self)
+    call_openai_api(docs, full_path, id, self)
     tokens = count_tokens_docs(docs)
     self.update_state(state="PROGRESS", meta={"current": 100})
 
@@ -138,24 +171,13 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         for i in range(min(5, len(raw_docs))):
             logging.info(f"Sample document {i}: {raw_docs[i]}")
 
-    # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
-    # and send them to the server (provide user and name in form)
-    file_data = {"name": name_job, "user": user, "tokens":tokens}
-    if settings.VECTOR_STORE == "faiss":
-        files = {
-            "file_faiss": open(full_path + "/index.faiss", "rb"),
-            "file_pkl": open(full_path + "/index.pkl", "rb"),
-        }
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
-        )
-        response = requests.get(
-            urljoin(settings.API_URL, "/api/delete_old?path=" + full_path)
-        )
-    else:
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), data=file_data
-        )
+    file_data.update({
+        "tokens": tokens,
+        "retriever": retriever,
+        "id": str(id),
+        "type": "local",
+    })
+    upload_index(full_path, file_data)
 
     # delete local
     shutil.rmtree(full_path)
@@ -169,47 +191,114 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         "limited": False,
     }
 
-
-def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
+def remote_worker(
+    self,
+    source_data,
+    name_job,
+    user,
+    loader,
+    directory="temp",
+    retriever="classic",
+    sync_frequency="never",
+    operation_mode="upload",
+    doc_id=None,
+):
     token_check = True
-    min_tokens = 150
-    max_tokens = 1250
-    full_path = directory + "/" + user + "/" + name_job
+    full_path = os.path.join(directory, user, name_job)
 
     if not os.path.exists(full_path):
         os.makedirs(full_path)
     self.update_state(state="PROGRESS", meta={"current": 1})
-    logging.info(f"Remote job: {full_path}", extra={"user": user, "job": name_job, source_data: source_data})
+    logging.info(
+        f"Remote job: {full_path}",
+        extra={"user": user, "job": name_job, "source_data": source_data},
+    )
 
     remote_loader = RemoteCreator.create_loader(loader)
     raw_docs = remote_loader.load_data(source_data)
 
     docs = group_split(
         documents=raw_docs,
-        min_tokens=min_tokens,
-        max_tokens=max_tokens,
+        min_tokens=MIN_TOKENS,
+        max_tokens=MAX_TOKENS,
         token_check=token_check,
     )
-    # docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
-    call_openai_api(docs, full_path, self)
     tokens = count_tokens_docs(docs)
+    if operation_mode == "upload":
+        id = ObjectId()
+        call_openai_api(docs, full_path, id, self)
+    elif operation_mode == "sync":
+        if not doc_id or not ObjectId.is_valid(doc_id):
+            raise ValueError("doc_id must be provided for sync operation.")
+        id = ObjectId(doc_id)
+        call_openai_api(docs, full_path, id, self)
     self.update_state(state="PROGRESS", meta={"current": 100})
 
-    # Proceed with uploading and cleaning as in the original function
-    file_data = {"name": name_job, "user": user, "tokens":tokens}
-    if settings.VECTOR_STORE == "faiss":
-        files = {
-            "file_faiss": open(full_path + "/index.faiss", "rb"),
-            "file_pkl": open(full_path + "/index.pkl", "rb"),
-        }
-        
-        requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
-        )
-        requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path))
-    else:
-        requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
+    file_data = {
+        "name": name_job,
+        "user": user,
+        "tokens": tokens,
+        "retriever": retriever,
+        "id": str(id),
+        "type": loader,
+        "remote_data": source_data,
+        "sync_frequency": sync_frequency,
+    }
+    upload_index(full_path, file_data)
 
     shutil.rmtree(full_path)
 
-    return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}
+    return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}
+
+def sync(
+    self,
+    source_data,
+    name_job,
+    user,
+    loader,
+    sync_frequency,
+    retriever,
+    doc_id=None,
+    directory="temp",
+):
+    try:
+        remote_worker(
+            self,
+            source_data,
+            name_job,
+            user,
+            loader,
+            directory,
+            retriever,
+            sync_frequency,
+            "sync",
+            doc_id,
+        )
+    except Exception as e:
+        logging.error(f"Error during sync: {e}")
+        return {"status": "error", "error": str(e)}
+    return {"status": "success"}
+
+def sync_worker(self, frequency):
+    sync_counts = Counter()
+    sources = sources_collection.find()
+    for doc in sources:
+        if doc.get("sync_frequency") == frequency:
+            name = doc.get("name")
+            user = doc.get("user")
+            source_type = doc.get("type")
+            source_data = doc.get("remote_data")
+            retriever = doc.get("retriever")
+            doc_id = str(doc.get("_id"))
+            resp = sync(
+                self, source_data, name, user, source_type, frequency, retriever, doc_id
+            )
+            sync_counts["total_sync_count"] += 1
+            sync_counts[
+                "sync_success" if resp["status"] == "success" else "sync_failure"
+            ] += 1
+
+    return {
+        key: sync_counts[key]
+        for key in ["total_sync_count", "sync_success", "sync_failure"]
+    }

docker-compose-local.yaml

@@ -1,6 +1,8 @@
 services:
   frontend:
     build: ./frontend
+    volumes:
+    - ./frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING

docker-compose.yaml

@@ -1,6 +1,8 @@
 services:
   frontend:
     build: ./frontend
+    volumes:
+      - ./frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING
@@ -18,6 +20,7 @@ services:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - CACHE_REDIS_URL=redis://redis:6379/2
     ports:
       - "7091:7091"
     volumes:
@@ -30,7 +33,7 @@ services:
 
   worker:
     build: ./application
-    command: celery -A application.app.celery worker -l INFO
+    command: celery -A application.app.celery worker -l INFO -B
     environment:
       - API_KEY=$API_KEY
       - EMBEDDINGS_KEY=$API_KEY
@@ -39,6 +42,7 @@ services:
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
+      - CACHE_REDIS_URL=redis://redis:6379/2
     depends_on:
       - redis
       - mongo

docs/README.md

@@ -46,6 +46,6 @@ yarn install
 yarn dev
 ```
 
-- Now, you should be able to view the docs on your local environment by visiting `http://localhost:5000`. You can explore the different markdown files and make changes as you see fit.
+- Now, you should be able to view the docs on your local environment by visiting `http://localhost:3000`. You can explore the different markdown files and make changes as you see fit.
 
 - **Footnotes:** This guide assumes you have Node.js and npm installed. The guide involves running a local server using yarn, and viewing the documentation offline. If you encounter any issues, it may be worth verifying your Node.js and npm installations and whether you have installed yarn correctly.

docs/package-lock.json

@@ -8,7 +8,7 @@
       "dependencies": {
         "@vercel/analytics": "^1.1.1",
         "docsgpt": "^0.4.1",
-        "next": "^14.1.1",
+        "next": "^14.2.12",
         "nextra": "^2.13.2",
         "nextra-theme-docs": "^2.13.2",
         "react": "^18.2.0",
@@ -936,14 +936,14 @@
       }
     },
     "node_modules/@next/env": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.1.tgz",
-      "integrity": "sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA=="
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.12.tgz",
+      "integrity": "sha512-3fP29GIetdwVIfIRyLKM7KrvJaqepv+6pVodEbx0P5CaMLYBtx+7eEg8JYO5L9sveJO87z9eCReceZLi0hxO1Q=="
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz",
-      "integrity": "sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.12.tgz",
+      "integrity": "sha512-crHJ9UoinXeFbHYNok6VZqjKnd8rTd7K3Z2zpyzF1ch7vVNKmhjv/V7EHxep3ILoN8JB9AdRn/EtVVyG9AkCXw==",
       "cpu": [
         "arm64"
       ],
@@ -956,9 +956,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz",
-      "integrity": "sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.12.tgz",
+      "integrity": "sha512-JbEaGbWq18BuNBO+lCtKfxl563Uw9oy2TodnN2ioX00u7V1uzrsSUcg3Ep9ce+P0Z9es+JmsvL2/rLphz+Frcw==",
       "cpu": [
         "x64"
       ],
@@ -971,9 +971,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz",
-      "integrity": "sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.12.tgz",
+      "integrity": "sha512-qBy7OiXOqZrdp88QEl2H4fWalMGnSCrr1agT/AVDndlyw2YJQA89f3ttR/AkEIP9EkBXXeGl6cC72/EZT5r6rw==",
       "cpu": [
         "arm64"
       ],
@@ -986,9 +986,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz",
-      "integrity": "sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.12.tgz",
+      "integrity": "sha512-EfD9L7o9biaQxjwP1uWXnk3vYZi64NVcKUN83hpVkKocB7ogJfyH2r7o1pPnMtir6gHZiGCeHKagJ0yrNSLNHw==",
       "cpu": [
         "arm64"
       ],
@@ -1001,9 +1001,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz",
-      "integrity": "sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.12.tgz",
+      "integrity": "sha512-iQ+n2pxklJew9IpE47hE/VgjmljlHqtcD5UhZVeHICTPbLyrgPehaKf2wLRNjYH75udroBNCgrSSVSVpAbNoYw==",
       "cpu": [
         "x64"
       ],
@@ -1016,9 +1016,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz",
-      "integrity": "sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.12.tgz",
+      "integrity": "sha512-rFkUkNwcQ0ODn7cxvcVdpHlcOpYxMeyMfkJuzaT74xjAa5v4fxP4xDk5OoYmPi8QNLDs3UgZPMSBmpBuv9zKWA==",
       "cpu": [
         "x64"
       ],
@@ -1031,9 +1031,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz",
-      "integrity": "sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.12.tgz",
+      "integrity": "sha512-PQFYUvwtHs/u0K85SG4sAdDXYIPXpETf9mcEjWc0R4JmjgMKSDwIU/qfZdavtP6MPNiMjuKGXHCtyhR/M5zo8g==",
       "cpu": [
         "arm64"
       ],
@@ -1046,9 +1046,9 @@
       }
     },
     "node_modules/@next/swc-win32-ia32-msvc": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz",
-      "integrity": "sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.12.tgz",
+      "integrity": "sha512-FAj2hMlcbeCV546eU2tEv41dcJb4NeqFlSXU/xL/0ehXywHnNpaYajOUvn3P8wru5WyQe6cTZ8fvckj/2XN4Vw==",
       "cpu": [
         "ia32"
       ],
@@ -1061,9 +1061,9 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz",
-      "integrity": "sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.12.tgz",
+      "integrity": "sha512-yu8QvV53sBzoIVRHsxCHqeuS8jYq6Lrmdh0briivuh+Brsp6xjg80MAozUsBTAV9KNmY08KlX0KYTWz1lbPzEg==",
       "cpu": [
         "x64"
       ],
@@ -1175,6 +1175,58 @@
         "node": ">=8"
       }
     },
+    "node_modules/@parcel/core": {
+      "version": "2.12.0",
+      "resolved": "https://registry.npmjs.org/@parcel/core/-/core-2.12.0.tgz",
+      "integrity": "sha512-s+6pwEj+GfKf7vqGUzN9iSEPueUssCCQrCBUlcAfKrJe0a22hTUCjewpB0I7lNrCIULt8dkndD+sMdOrXsRl6Q==",
+      "peer": true,
+      "dependencies": {
+        "@mischnic/json-sourcemap": "^0.1.0",
+        "@parcel/cache": "2.12.0",
+        "@parcel/diagnostic": "2.12.0",
+        "@parcel/events": "2.12.0",
+        "@parcel/fs": "2.12.0",
+        "@parcel/graph": "3.2.0",
+        "@parcel/logger": "2.12.0",
+        "@parcel/package-manager": "2.12.0",
+        "@parcel/plugin": "2.12.0",
+        "@parcel/profiler": "2.12.0",
+        "@parcel/rust": "2.12.0",
+        "@parcel/source-map": "^2.1.1",
+        "@parcel/types": "2.12.0",
+        "@parcel/utils": "2.12.0",
+        "@parcel/workers": "2.12.0",
+        "abortcontroller-polyfill": "^1.1.9",
+        "base-x": "^3.0.8",
+        "browserslist": "^4.6.6",
+        "clone": "^2.1.1",
+        "dotenv": "^7.0.0",
+        "dotenv-expand": "^5.1.0",
+        "json5": "^2.2.0",
+        "msgpackr": "^1.9.9",
+        "nullthrows": "^1.1.1",
+        "semver": "^7.5.2"
+      },
+      "engines": {
+        "node": ">= 12.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@parcel/core/node_modules/semver": {
+      "version": "7.6.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
+      "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
+      "peer": true,
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/@parcel/diagnostic": {
       "version": "2.12.0",
       "resolved": "https://registry.npmjs.org/@parcel/diagnostic/-/diagnostic-2.12.0.tgz",
@@ -1225,6 +1277,22 @@
         "@parcel/core": "^2.12.0"
       }
     },
+    "node_modules/@parcel/graph": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/@parcel/graph/-/graph-3.2.0.tgz",
+      "integrity": "sha512-xlrmCPqy58D4Fg5umV7bpwDx5Vyt7MlnQPxW68vae5+BA4GSWetfZt+Cs5dtotMG2oCHzZxhIPt7YZ7NRyQzLA==",
+      "peer": true,
+      "dependencies": {
+        "nullthrows": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 12.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
     "node_modules/@parcel/logger": {
       "version": "2.12.0",
       "resolved": "https://registry.npmjs.org/@parcel/logger/-/logger-2.12.0.tgz",
@@ -2476,10 +2544,11 @@
       "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="
     },
     "node_modules/@swc/helpers": {
-      "version": "0.5.2",
-      "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.2.tgz",
-      "integrity": "sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==",
+      "version": "0.5.5",
+      "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz",
+      "integrity": "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==",
       "dependencies": {
+        "@swc/counter": "^0.1.3",
         "tslib": "^2.4.0"
       }
     },
@@ -2648,6 +2717,12 @@
         "server-only": "^0.0.1"
       }
     },
+    "node_modules/abortcontroller-polyfill": {
+      "version": "1.7.5",
+      "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.5.tgz",
+      "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ==",
+      "peer": true
+    },
     "node_modules/acorn": {
       "version": "8.11.3",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
@@ -2727,6 +2802,15 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/base-x": {
+      "version": "3.0.10",
+      "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.10.tgz",
+      "integrity": "sha512-7d0s06rR9rYaIWHkpfLIFICM/tkSVdoPC9qYAQRpxn9DdKNWNsKC0uk++akckyLq16Tx2WIinnZ6WRriAt6njQ==",
+      "peer": true,
+      "dependencies": {
+        "safe-buffer": "^5.0.1"
+      }
+    },
     "node_modules/boolbase": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
@@ -2941,6 +3025,15 @@
         "node": ">=4"
       }
     },
+    "node_modules/clone": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
+      "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==",
+      "peer": true,
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/clsx": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.0.tgz",
@@ -3693,6 +3786,21 @@
         "url": "https://github.com/fb55/domutils?sponsor=1"
       }
     },
+    "node_modules/dotenv": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-7.0.0.tgz",
+      "integrity": "sha512-M3NhsLbV1i6HuGzBUH8vXrtxOk+tWmzWKDMbAVSUp3Zsjm7ywFeuwrUXhmhQyRK1q5B5GGy7hcXPbj3bnfZg2g==",
+      "peer": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/dotenv-expand": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz",
+      "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==",
+      "peer": true
+    },
     "node_modules/electron-to-chromium": {
       "version": "1.4.693",
       "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.693.tgz",
@@ -6241,12 +6349,12 @@
       }
     },
     "node_modules/next": {
-      "version": "14.1.1",
-      "resolved": "https://registry.npmjs.org/next/-/next-14.1.1.tgz",
-      "integrity": "sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww==",
+      "version": "14.2.12",
+      "resolved": "https://registry.npmjs.org/next/-/next-14.2.12.tgz",
+      "integrity": "sha512-cDOtUSIeoOvt1skKNihdExWMTybx3exnvbFbb9ecZDIxlvIbREQzt9A5Km3Zn3PfU+IFjyYGsHS+lN9VInAGKA==",
       "dependencies": {
-        "@next/env": "14.1.1",
-        "@swc/helpers": "0.5.2",
+        "@next/env": "14.2.12",
+        "@swc/helpers": "0.5.5",
         "busboy": "1.6.0",
         "caniuse-lite": "^1.0.30001579",
         "graceful-fs": "^4.2.11",
@@ -6260,18 +6368,19 @@
         "node": ">=18.17.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "14.1.1",
-        "@next/swc-darwin-x64": "14.1.1",
-        "@next/swc-linux-arm64-gnu": "14.1.1",
-        "@next/swc-linux-arm64-musl": "14.1.1",
-        "@next/swc-linux-x64-gnu": "14.1.1",
-        "@next/swc-linux-x64-musl": "14.1.1",
-        "@next/swc-win32-arm64-msvc": "14.1.1",
-        "@next/swc-win32-ia32-msvc": "14.1.1",
-        "@next/swc-win32-x64-msvc": "14.1.1"
+        "@next/swc-darwin-arm64": "14.2.12",
+        "@next/swc-darwin-x64": "14.2.12",
+        "@next/swc-linux-arm64-gnu": "14.2.12",
+        "@next/swc-linux-arm64-musl": "14.2.12",
+        "@next/swc-linux-x64-gnu": "14.2.12",
+        "@next/swc-linux-x64-musl": "14.2.12",
+        "@next/swc-win32-arm64-msvc": "14.2.12",
+        "@next/swc-win32-ia32-msvc": "14.2.12",
+        "@next/swc-win32-x64-msvc": "14.2.12"
       },
       "peerDependencies": {
         "@opentelemetry/api": "^1.1.0",
+        "@playwright/test": "^1.41.2",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
         "sass": "^1.3.0"
@@ -6280,6 +6389,9 @@
         "@opentelemetry/api": {
           "optional": true
         },
+        "@playwright/test": {
+          "optional": true
+        },
         "sass": {
           "optional": true
         }
@@ -9575,6 +9687,26 @@
         "node": ">=6"
       }
     },
+    "node_modules/safe-buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "peer": true
+    },
     "node_modules/safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
@@ -9919,6 +10051,19 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/typescript": {
+      "version": "5.6.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz",
+      "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==",
+      "peer": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    },
     "node_modules/unified": {
       "version": "10.1.2",
       "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.2.tgz",

docs/package.json

@@ -8,7 +8,7 @@
   "dependencies": {
     "@vercel/analytics": "^1.1.1",
     "docsgpt": "^0.4.1",
-    "next": "^14.1.1",
+    "next": "^14.2.12",
     "nextra": "^2.13.2",
     "nextra-theme-docs": "^2.13.2",
     "react": "^18.2.0",

docs/pages/Deploying/Quickstart.md

@@ -67,46 +67,3 @@ To run the setup on Windows, you have two options: using the Windows Subsystem f
 
 These steps should help you set up and run the project on Windows using either WSL or Git Bash/Command Prompt. 
 **Important:** Ensure that Docker is installed and properly configured on your Windows system for these steps to work.
-
-
-For WINDOWS:
-
-To run the given setup on Windows, you can use the Windows Subsystem for Linux (WSL) or a Git Bash terminal to execute similar commands. Here are the steps adapted for Windows:
-
-Option 1: Using Windows Subsystem for Linux (WSL):
-
-1. Install WSL if you haven't already. You can follow the official Microsoft documentation for installation: (https://learn.microsoft.com/en-us/windows/wsl/install).
-2. After setting up WSL, open the WSL terminal.
-3. Clone the repository and create the `.env` file:
-   ```bash
-   git clone https://github.com/arc53/DocsGPT.git
-   cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
-   ```
-4. Run the following command to start the setup with Docker Compose:
-    ```bash
-    ./run-with-docker-compose.sh
-    ```
-5. Open your web browser and navigate to http://localhost:5173/.
-6. To stop the setup, just press **Ctrl + C** in the WSL terminal.
-
-Option 2: Using Git Bash or Command Prompt (CMD):
-
-1. Install Git for Windows if you haven't already. You can download it from the official website: (https://gitforwindows.org/).
-2. Open Git Bash or Command Prompt.
-3. Clone the repository and create the `.env` file:
-   ```bash
-   git clone https://github.com/arc53/DocsGPT.git
-   cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
-   ```
-4. Run the following command to start the setup with Docker Compose:
-   ```bash
-   ./run-with-docker-compose.sh
-   ```
-5. Open your web browser and navigate to http://localhost:5173/.
-6. To stop the setup, just press **Ctrl + C** in the Git Bash or Command Prompt terminal.
-
-These steps should help you set up and run the project on Windows using either WSL or Git Bash/Command Prompt. Make sure you have Docker installed and properly configured on your Windows system for this to work.

docs/pages/Guides/How-to-train-on-other-documentation.mdx

@@ -28,15 +28,15 @@ Navigate to the sidebar where you will find `Source Docs` option,here you will f
 
 
 ### Step 2
-Click on the `Upload icon` just beside the source docs options,now borwse and upload the document which you want to train on or select the `remote` option if you have to insert the link of the documentation.
+Click on the `Upload icon` just beside the source docs options,now browse and upload the document which you want to train on or select the `remote` option if you have to insert the link of the documentation.
 
 
 ### Step 3
-Now you will be able to see the name of the file uploaded under the Uploaded Files ,now click on `Train`,once you click on train it might take some time to train on the document. You will be able to see the `Training progress` and once the training is completed you can click the `finish` button and there you go your docuemnt is uploaded.
+Now you will be able to see the name of the file uploaded under the Uploaded Files ,now click on `Train`,once you click on train it might take some time to train on the document. You will be able to see the `Training progress` and once the training is completed you can click the `finish` button and there you go your document is uploaded.
 
 
 ### Step 4
-Go to `New chat`  and from the side bar select the document  you uploaded  under the `Source Docs` and go ahead with your chat, now you can ask qestions regarding the document you uploaded and you will get the effective answer based on it.
+Go to `New chat`  and from the side bar select the document  you uploaded  under the `Source Docs` and go ahead with your chat, now you can ask questions regarding the document you uploaded and you will get the effective answer based on it.
 
 </Steps>
 

docs/pages/Guides/How-to-use-different-LLM.mdx

@@ -33,7 +33,7 @@ For open source you  have to edit .env file with LLM_NAME with their desired LLM
 All the supported LLM providers are here application/llm and you  can check what env variable are needed for each
 List of latest supported LLMs are https://github.com/arc53/DocsGPT/blob/main/application/llm/llm_creator.py
 ### Step 3
-Visit application/llm and select the file of your selected llm and there you will find the speicifc requirements needed to be filled in order to use it,i.e API key of that llm.
+Visit application/llm and select the file of your selected llm and there you will find the specific requirements needed to be filled in order to use it,i.e API key of that llm.
 </Steps>
 
 ### For OpenAI-Compatible Endpoints:

extensions/react-widget/package-lock.json

@@ -1885,6 +1885,17 @@
         "node": ">=6.0.0"
       }
     },
+    "node_modules/@jridgewell/source-map": {
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz",
+      "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@jridgewell/gen-mapping": "^0.3.5",
+        "@jridgewell/trace-mapping": "^0.3.25"
+      }
+    },
     "node_modules/@jridgewell/sourcemap-codec": {
       "version": "1.4.15",
       "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
@@ -2258,7 +2269,6 @@
       "version": "2.12.0",
       "resolved": "https://registry.npmjs.org/@parcel/core/-/core-2.12.0.tgz",
       "integrity": "sha512-s+6pwEj+GfKf7vqGUzN9iSEPueUssCCQrCBUlcAfKrJe0a22hTUCjewpB0I7lNrCIULt8dkndD+sMdOrXsRl6Q==",
-      "dev": true,
       "dependencies": {
         "@mischnic/json-sourcemap": "^0.1.0",
         "@parcel/cache": "2.12.0",
@@ -2298,7 +2308,6 @@
       "version": "7.6.2",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz",
       "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==",
-      "dev": true,
       "bin": {
         "semver": "bin/semver.js"
       },
@@ -2360,7 +2369,6 @@
       "version": "3.2.0",
       "resolved": "https://registry.npmjs.org/@parcel/graph/-/graph-3.2.0.tgz",
       "integrity": "sha512-xlrmCPqy58D4Fg5umV7bpwDx5Vyt7MlnQPxW68vae5+BA4GSWetfZt+Cs5dtotMG2oCHzZxhIPt7YZ7NRyQzLA==",
-      "dev": true,
       "dependencies": {
         "nullthrows": "^1.1.1"
       },
@@ -4560,7 +4568,7 @@
       "version": "0.5.11",
       "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.11.tgz",
       "integrity": "sha512-YNlnKRWF2sVojTpIyzwou9XoTNbzbzONwRhOoniEioF1AtaitTvVZblaQRrAzChWQ1bLYyYSWzM18y4WwgzJ+A==",
-      "dev": true,
+      "devOptional": true,
       "dependencies": {
         "tslib": "^2.4.0"
       }
@@ -4590,6 +4598,13 @@
         "@types/trusted-types": "*"
       }
     },
+    "node_modules/@types/estree": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
+      "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.15",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@@ -4621,6 +4636,16 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/node": {
+      "version": "22.5.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.5.tgz",
+      "integrity": "sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "undici-types": "~6.19.2"
+      }
+    },
     "node_modules/@types/parse-json": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz",
@@ -4662,11 +4687,208 @@
       "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
       "dev": true
     },
+    "node_modules/@webassemblyjs/ast": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz",
+      "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/helper-numbers": "1.11.6",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6"
+      }
+    },
+    "node_modules/@webassemblyjs/floating-point-hex-parser": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz",
+      "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==",
+      "dev": true,
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/helper-api-error": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz",
+      "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==",
+      "dev": true,
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/helper-buffer": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz",
+      "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw==",
+      "dev": true,
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/helper-numbers": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz",
+      "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/floating-point-hex-parser": "1.11.6",
+        "@webassemblyjs/helper-api-error": "1.11.6",
+        "@xtuc/long": "4.2.2"
+      }
+    },
+    "node_modules/@webassemblyjs/helper-wasm-bytecode": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz",
+      "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==",
+      "dev": true,
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/helper-wasm-section": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz",
+      "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-buffer": "1.12.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+        "@webassemblyjs/wasm-gen": "1.12.1"
+      }
+    },
+    "node_modules/@webassemblyjs/ieee754": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz",
+      "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@xtuc/ieee754": "^1.2.0"
+      }
+    },
+    "node_modules/@webassemblyjs/leb128": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz",
+      "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@xtuc/long": "4.2.2"
+      }
+    },
+    "node_modules/@webassemblyjs/utf8": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz",
+      "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==",
+      "dev": true,
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/wasm-edit": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz",
+      "integrity": "sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-buffer": "1.12.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+        "@webassemblyjs/helper-wasm-section": "1.12.1",
+        "@webassemblyjs/wasm-gen": "1.12.1",
+        "@webassemblyjs/wasm-opt": "1.12.1",
+        "@webassemblyjs/wasm-parser": "1.12.1",
+        "@webassemblyjs/wast-printer": "1.12.1"
+      }
+    },
+    "node_modules/@webassemblyjs/wasm-gen": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz",
+      "integrity": "sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+        "@webassemblyjs/ieee754": "1.11.6",
+        "@webassemblyjs/leb128": "1.11.6",
+        "@webassemblyjs/utf8": "1.11.6"
+      }
+    },
+    "node_modules/@webassemblyjs/wasm-opt": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz",
+      "integrity": "sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-buffer": "1.12.1",
+        "@webassemblyjs/wasm-gen": "1.12.1",
+        "@webassemblyjs/wasm-parser": "1.12.1"
+      }
+    },
+    "node_modules/@webassemblyjs/wasm-parser": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz",
+      "integrity": "sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-api-error": "1.11.6",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+        "@webassemblyjs/ieee754": "1.11.6",
+        "@webassemblyjs/leb128": "1.11.6",
+        "@webassemblyjs/utf8": "1.11.6"
+      }
+    },
+    "node_modules/@webassemblyjs/wast-printer": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz",
+      "integrity": "sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@xtuc/long": "4.2.2"
+      }
+    },
+    "node_modules/@xtuc/ieee754": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",
+      "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==",
+      "dev": true,
+      "peer": true
+    },
+    "node_modules/@xtuc/long": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz",
+      "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/abortcontroller-polyfill": {
       "version": "1.7.5",
       "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.5.tgz",
-      "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ==",
-      "dev": true
+      "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ=="
+    },
+    "node_modules/acorn": {
+      "version": "8.12.1",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
+      "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==",
+      "dev": true,
+      "peer": true,
+      "bin": {
+        "acorn": "bin/acorn"
+      },
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/acorn-import-attributes": {
+      "version": "1.9.5",
+      "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz",
+      "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==",
+      "dev": true,
+      "peer": true,
+      "peerDependencies": {
+        "acorn": "^8"
+      }
     },
     "node_modules/ajv": {
       "version": "6.12.6",
@@ -4771,7 +4993,6 @@
       "version": "3.0.9",
       "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.9.tgz",
       "integrity": "sha512-H7JU6iBHTal1gp56aKoaa//YUxEaAOUiydvrV/pILqIHXTtqxSkATOnDA2u+jZ/61sD+L/412+7kzXRtWukhpQ==",
-      "dev": true,
       "dependencies": {
         "safe-buffer": "^5.0.1"
       }
@@ -4832,6 +5053,13 @@
         "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
       }
     },
+    "node_modules/buffer-from": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
+      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/callsites": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -4922,7 +5150,6 @@
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
       "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==",
-      "dev": true,
       "engines": {
         "node": ">=0.8"
       }
@@ -5132,7 +5359,6 @@
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-7.0.0.tgz",
       "integrity": "sha512-M3NhsLbV1i6HuGzBUH8vXrtxOk+tWmzWKDMbAVSUp3Zsjm7ywFeuwrUXhmhQyRK1q5B5GGy7hcXPbj3bnfZg2g==",
-      "dev": true,
       "engines": {
         "node": ">=6"
       }
@@ -5140,8 +5366,7 @@
     "node_modules/dotenv-expand": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz",
-      "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==",
-      "dev": true
+      "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA=="
     },
     "node_modules/electron-to-chromium": {
       "version": "1.4.788",
@@ -5157,6 +5382,20 @@
         "node": ">= 4"
       }
     },
+    "node_modules/enhanced-resolve": {
+      "version": "5.17.1",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz",
+      "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "graceful-fs": "^4.2.4",
+        "tapable": "^2.2.0"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    },
     "node_modules/entities": {
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
@@ -5185,6 +5424,13 @@
         "is-arrayish": "^0.2.1"
       }
     },
+    "node_modules/es-module-lexer": {
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
+      "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/escalade": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz",
@@ -5201,6 +5447,53 @@
         "node": ">=0.8.0"
       }
     },
+    "node_modules/eslint-scope": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
+      "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "esrecurse": "^4.3.0",
+        "estraverse": "^4.1.1"
+      },
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/esrecurse": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+      "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "estraverse": "^5.2.0"
+      },
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
+    "node_modules/esrecurse/node_modules/estraverse": {
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
+      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+      "dev": true,
+      "peer": true,
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
+    "node_modules/estraverse": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
+      "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
+      "dev": true,
+      "peer": true,
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
     "node_modules/esutils": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
@@ -5210,6 +5503,16 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+      "dev": true,
+      "peer": true,
+      "engines": {
+        "node": ">=0.8.x"
+      }
+    },
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -5300,6 +5603,13 @@
         "node": ">=6"
       }
     },
+    "node_modules/glob-to-regexp": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
+      "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/globals": {
       "version": "11.12.0",
       "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
@@ -5308,6 +5618,13 @@
         "node": ">=4"
       }
     },
+    "node_modules/graceful-fs": {
+      "version": "4.2.11",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
+      "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -5513,6 +5830,47 @@
         "node": ">=0.12.0"
       }
     },
+    "node_modules/jest-worker": {
+      "version": "27.5.1",
+      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz",
+      "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@types/node": "*",
+        "merge-stream": "^2.0.0",
+        "supports-color": "^8.0.0"
+      },
+      "engines": {
+        "node": ">= 10.13.0"
+      }
+    },
+    "node_modules/jest-worker/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "peer": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-worker/node_modules/supports-color": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
+      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/supports-color?sponsor=1"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -5813,6 +6171,16 @@
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
       "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="
     },
+    "node_modules/loader-runner": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
+      "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==",
+      "dev": true,
+      "peer": true,
+      "engines": {
+        "node": ">=6.11.5"
+      }
+    },
     "node_modules/loader-utils": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz",
@@ -5902,10 +6270,17 @@
       "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==",
       "license": "MIT"
     },
+    "node_modules/merge-stream": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
+      "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/micromatch": {
-      "version": "4.0.7",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
-      "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
+      "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
       "dependencies": {
         "braces": "^3.0.3",
         "picomatch": "^2.3.1"
@@ -5914,6 +6289,29 @@
         "node": ">=8.6"
       }
     },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "dev": true,
+      "peer": true,
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -5976,6 +6374,13 @@
         "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
       }
     },
+    "node_modules/neo-async": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
+      "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/node-addon-api": {
       "version": "7.1.0",
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.0.tgz",
@@ -8750,6 +9155,16 @@
         "node": ">=6"
       }
     },
+    "node_modules/randombytes": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
+      "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "safe-buffer": "^5.1.0"
+      }
+    },
     "node_modules/react": {
       "version": "18.3.1",
       "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
@@ -8888,7 +9303,6 @@
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
       "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
-      "dev": true,
       "funding": [
         {
           "type": "github",
@@ -8938,6 +9352,16 @@
         "semver": "bin/semver.js"
       }
     },
+    "node_modules/serialize-javascript": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz",
+      "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "randombytes": "^2.1.0"
+      }
+    },
     "node_modules/shallowequal": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz",
@@ -8959,6 +9383,17 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/source-map-support": {
+      "version": "0.5.21",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz",
+      "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "source-map": "^0.6.0"
+      }
+    },
     "node_modules/stable": {
       "version": "0.1.8",
       "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz",
@@ -9025,6 +9460,16 @@
       "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz",
       "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ=="
     },
+    "node_modules/tapable": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
+      "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==",
+      "dev": true,
+      "peer": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/term-size": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz",
@@ -9037,6 +9482,86 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/terser": {
+      "version": "5.33.0",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.33.0.tgz",
+      "integrity": "sha512-JuPVaB7s1gdFKPKTelwUyRq5Sid2A3Gko2S0PncwdBq7kN9Ti9HPWDQ06MPsEDGsZeVESjKEnyGy68quBk1w6g==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@jridgewell/source-map": "^0.3.3",
+        "acorn": "^8.8.2",
+        "commander": "^2.20.0",
+        "source-map-support": "~0.5.20"
+      },
+      "bin": {
+        "terser": "bin/terser"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/terser-webpack-plugin": {
+      "version": "5.3.10",
+      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz",
+      "integrity": "sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@jridgewell/trace-mapping": "^0.3.20",
+        "jest-worker": "^27.4.5",
+        "schema-utils": "^3.1.1",
+        "serialize-javascript": "^6.0.1",
+        "terser": "^5.26.0"
+      },
+      "engines": {
+        "node": ">= 10.13.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependencies": {
+        "webpack": "^5.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@swc/core": {
+          "optional": true
+        },
+        "esbuild": {
+          "optional": true
+        },
+        "uglify-js": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/terser-webpack-plugin/node_modules/schema-utils": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz",
+      "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@types/json-schema": "^7.0.8",
+        "ajv": "^6.12.5",
+        "ajv-keywords": "^3.5.2"
+      },
+      "engines": {
+        "node": ">= 10.13.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      }
+    },
+    "node_modules/terser/node_modules/commander": {
+      "version": "2.20.3",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/timsort": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/timsort/-/timsort-0.3.0.tgz",
@@ -9083,7 +9608,6 @@
       "version": "5.4.5",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz",
       "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==",
-      "dev": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -9098,6 +9622,13 @@
       "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
       "license": "MIT"
     },
+    "node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "dev": true,
+      "peer": true
+    },
     "node_modules/unicode-canonical-property-names-ecmascript": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz",
@@ -9184,11 +9715,101 @@
         "node": ">= 4"
       }
     },
+    "node_modules/watchpack": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz",
+      "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "glob-to-regexp": "^0.4.1",
+        "graceful-fs": "^4.1.2"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    },
     "node_modules/weak-lru-cache": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz",
       "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw=="
     },
+    "node_modules/webpack": {
+      "version": "5.94.0",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.94.0.tgz",
+      "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@types/estree": "^1.0.5",
+        "@webassemblyjs/ast": "^1.12.1",
+        "@webassemblyjs/wasm-edit": "^1.12.1",
+        "@webassemblyjs/wasm-parser": "^1.12.1",
+        "acorn": "^8.7.1",
+        "acorn-import-attributes": "^1.9.5",
+        "browserslist": "^4.21.10",
+        "chrome-trace-event": "^1.0.2",
+        "enhanced-resolve": "^5.17.1",
+        "es-module-lexer": "^1.2.1",
+        "eslint-scope": "5.1.1",
+        "events": "^3.2.0",
+        "glob-to-regexp": "^0.4.1",
+        "graceful-fs": "^4.2.11",
+        "json-parse-even-better-errors": "^2.3.1",
+        "loader-runner": "^4.2.0",
+        "mime-types": "^2.1.27",
+        "neo-async": "^2.6.2",
+        "schema-utils": "^3.2.0",
+        "tapable": "^2.1.1",
+        "terser-webpack-plugin": "^5.3.10",
+        "watchpack": "^2.4.1",
+        "webpack-sources": "^3.2.3"
+      },
+      "bin": {
+        "webpack": "bin/webpack.js"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependenciesMeta": {
+        "webpack-cli": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/webpack-sources": {
+      "version": "3.2.3",
+      "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz",
+      "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==",
+      "dev": true,
+      "peer": true,
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    },
+    "node_modules/webpack/node_modules/schema-utils": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz",
+      "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@types/json-schema": "^7.0.8",
+        "ajv": "^6.12.5",
+        "ajv-keywords": "^3.5.2"
+      },
+      "engines": {
+        "node": ">= 10.13.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      }
+    },
     "node_modules/yallist": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",

extensions/react-widget/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "docsgpt-react",
+  "name": "docsgpt",
   "version": "0.4.2",
   "private": false,
   "description": "DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.",
@@ -11,6 +11,18 @@
     "dist",
     "package.json"
   ],
+  "targets": {
+    "modern": {
+      "engines": {
+        "browsers": "Chrome 80"
+      }
+    },
+    "legacy": {
+      "engines": {
+        "browsers": "> 0.5%, last 2 versions, not dead"
+      }
+    }
+  },
   "@parcel/resolver-default": {
     "packageExports": true
   },

extensions/react-widget/src/assets/dislike.svg

@@ -0,0 +1,4 @@
+<svg width="14" height="14" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.37776 10.1001V12.9C6.37776 13.457 6.599 13.9911 6.99282 14.3849C7.38664 14.7788 7.92077 15 8.47772 15L11.2777 8.70011V1.00025H3.38181C3.04419 0.996436 2.71656 1.11477 2.45929 1.33344C2.20203 1.55212 2.03246 1.8564 1.98184 2.19023L1.01585 8.49012C0.985398 8.69076 0.998931 8.89563 1.05551 9.09053C1.1121 9.28543 1.21038 9.46569 1.34355 9.61884C1.47671 9.77198 1.64159 9.89434 1.82674 9.97744C2.01189 10.0605 2.2129 10.1024 2.41583 10.1001H6.37776ZM11.2777 1.00025H13.1466C13.5428 0.993247 13.9277 1.13195 14.2284 1.39002C14.5291 1.64809 14.7245 2.00758 14.7776 2.40023V7.30014C14.7245 7.69279 14.5291 8.05227 14.2284 8.31035C13.9277 8.56842 13.5428 8.70712 13.1466 8.70011H11.2777" fill="none"/>
+<path d="M11.2777 8.70011L8.47772 15C7.92077 15 7.38664 14.7788 6.99282 14.3849C6.599 13.9911 6.37776 13.457 6.37776 12.9V10.1001H2.41583C2.2129 10.1024 2.01189 10.0605 1.82674 9.97744C1.64159 9.89434 1.47671 9.77198 1.34355 9.61884C1.21038 9.46569 1.1121 9.28543 1.05551 9.09053C0.998931 8.89563 0.985398 8.69076 1.01585 8.49012L1.98184 2.19023C2.03246 1.8564 2.20203 1.55212 2.45929 1.33344C2.71656 1.11477 3.04419 0.996436 3.38181 1.00025H11.2777M11.2777 8.70011V1.00025M11.2777 8.70011H13.1466C13.5428 8.70712 13.9277 8.56842 14.2284 8.31035C14.5291 8.05227 14.7245 7.69279 14.7776 7.30014V2.40023C14.7245 2.00758 14.5291 1.64809 14.2284 1.39002C13.9277 1.13195 13.5428 0.993247 13.1466 1.00025H11.2777" stroke="current" stroke-width="1.4" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

extensions/react-widget/src/assets/like.svg

@@ -0,0 +1,4 @@
+<svg width="14" height="14" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
+<path d="M9.39995 5.89997V3.09999C9.39995 2.54304 9.1787 2.0089 8.78487 1.61507C8.39105 1.22125 7.85691 1 7.29996 1L4.49998 7.29996V14.9999H12.3959C12.7336 15.0037 13.0612 14.8854 13.3185 14.6667C13.5757 14.448 13.7453 14.1437 13.7959 13.8099L14.7619 7.50996C14.7924 7.30931 14.7788 7.10444 14.7222 6.90954C14.6657 6.71464 14.5674 6.53437 14.4342 6.38123C14.301 6.22808 14.1362 6.10572 13.951 6.02262C13.7659 5.93952 13.5649 5.89767 13.3619 5.89997H9.39995ZM4.49998 14.9999H2.39999C2.02869 14.9999 1.6726 14.8524 1.41005 14.5899C1.1475 14.3273 1 13.9712 1 13.5999V8.69995C1 8.32865 1.1475 7.97256 1.41005 7.71001C1.6726 7.44746 2.02869 7.29996 2.39999 7.29996H4.49998" fill="none"/>
+<path d="M4.49998 7.29996L7.29996 1C7.85691 1 8.39105 1.22125 8.78487 1.61507C9.1787 2.0089 9.39995 2.54304 9.39995 3.09999V5.89997H13.3619C13.5649 5.89767 13.7659 5.93952 13.951 6.02262C14.1362 6.10572 14.301 6.22808 14.4342 6.38123C14.5674 6.53437 14.6657 6.71464 14.7223 6.90954C14.7788 7.10444 14.7924 7.30931 14.7619 7.50996L13.7959 13.8099C13.7453 14.1437 13.5757 14.448 13.3185 14.6667C13.0612 14.8854 12.7336 15.0037 12.3959 14.9999H4.49998M4.49998 7.29996V14.9999M4.49998 7.29996H2.39999C2.02869 7.29996 1.6726 7.44746 1.41005 7.71001C1.1475 7.97256 1 8.32865 1 8.69995V13.5999C1 13.9712 1.1475 14.3273 1.41005 14.5899C1.6726 14.8524 2.02869 14.9999 2.39999 14.9999H4.49998" stroke="current" stroke-width="1.39999" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

extensions/react-widget/src/components/DocsGPTWidget.tsx

@@ -1,11 +1,13 @@
 "use client";
-import React from 'react'
+import React, { useRef } from 'react'
 import DOMPurify from 'dompurify';
 import styled, { keyframes, createGlobalStyle } from 'styled-components';
 import { PaperPlaneIcon, RocketIcon, ExclamationTriangleIcon, Cross2Icon } from '@radix-ui/react-icons';
-import { MESSAGE_TYPE, Query, Status, WidgetProps } from '../types/index';
-import { fetchAnswerStreaming } from '../requests/streamingApi';
+import { FEEDBACK, MESSAGE_TYPE, Query, Status, WidgetProps } from '../types/index';
+import { fetchAnswerStreaming, sendFeedback } from '../requests/streamingApi';
 import { ThemeProvider } from 'styled-components';
+import Like from "../assets/like.svg"
+import Dislike from "../assets/dislike.svg"
 import MarkdownIt from 'markdown-it';
 const themes = {
   dark: {
@@ -63,6 +65,10 @@ const GlobalStyles = createGlobalStyle`
   background-color: #646464;
   color: #fff !important;
 }
+.response code {
+  white-space: pre-wrap !important;
+  line-break: loose !important;
+}
 `;
 const Overlay = styled.div`
   position: fixed;
@@ -195,12 +201,24 @@ const Conversation = styled.div<{ size: string }>`
     width:${props => props.size === 'large' ? '90vw' : props.size === 'medium' ? '60vw' : '400px'} !important;
     }
 `;
-
+const Feedback = styled.div`
+  background-color: transparent;
+  font-weight: normal;
+  gap: 12px;
+  display: flex;
+  padding: 6px;
+  clear: both;
+`;
 const MessageBubble = styled.div<{ type: MESSAGE_TYPE }>`
-    display: flex;
+    display: block;
     font-size: 16px;
-    justify-content: ${props => props.type === 'QUESTION' ? 'flex-end' : 'flex-start'};
-    margin: 0.5rem;
+    position: relative;
+    width: 100%;;
+    float: right;
+    margin: 0rem;
+    &:hover ${Feedback} * {
+    visibility: visible !important;
+  }
 `;
 const Message = styled.div<{ type: MESSAGE_TYPE }>`
     background: ${props => props.type === 'QUESTION' ?
@@ -208,6 +226,7 @@ const Message = styled.div<{ type: MESSAGE_TYPE }>`
     props.theme.secondary.bg};
     color: ${props => props.type === 'ANSWER' ? props.theme.primary.text : '#fff'};
     border: none;
+    float: ${props => props.type === 'QUESTION' ? 'right' : 'left'};
     max-width: ${props => props.type === 'ANSWER' ? '100%' : '80'};
     overflow: auto;
     margin: 4px;
@@ -315,6 +334,7 @@ const HeroDescription = styled.p`
   font-size: 14px;
   line-height: 1.5;
 `;
+
 const Hero = ({ title, description, theme }: { title: string, description: string, theme: string }) => {
   return (
     <>
@@ -345,7 +365,8 @@ export const DocsGPTWidget = ({
   size = 'small',
   theme = 'dark',
   buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/message.svg',
-  buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)'
+  buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)',
+  collectFeedback = true
 }: WidgetProps) => {
   const [prompt, setPrompt] = React.useState('');
   const [status, setStatus] = React.useState<Status>('idle');
@@ -353,6 +374,7 @@ export const DocsGPTWidget = ({
   const [conversationId, setConversationId] = React.useState<string | null>(null)
   const [open, setOpen] = React.useState<boolean>(false)
   const [eventInterrupt, setEventInterrupt] = React.useState<boolean>(false); //click or scroll by user while autoScrolling
+  const isBubbleHovered = useRef<boolean>(false)
   const endMessageRef = React.useRef<HTMLDivElement | null>(null);
   const md = new MarkdownIt();
 
@@ -376,6 +398,36 @@ export const DocsGPTWidget = ({
     !eventInterrupt && scrollToBottom(endMessageRef.current);
   }, [queries.length, queries[queries.length - 1]?.response]);
 
+  async function handleFeedback(feedback: FEEDBACK, index: number) {
+    let query = queries[index]
+    if (!query.response)
+      return;
+    if (query.feedback != feedback) {
+      sendFeedback({
+        question: query.prompt,
+        answer: query.response,
+        feedback: feedback,
+        apikey: apiKey
+      }, apiHost)
+        .then(res => {
+          if (res.status == 200) {
+            query.feedback = feedback;
+            setQueries((prev: Query[]) => {
+              return prev.map((q, i) => (i === index ? query : q));
+            });
+          }
+        })
+        .catch(err => console.log("Connection failed",err))
+    }
+    else {
+      delete query.feedback;
+      setQueries((prev: Query[]) => {
+        return prev.map((q, i) => (i === index ? query : q));
+      });
+
+    }
+  }
+
   async function stream(question: string) {
     setStatus('loading')
     try {
@@ -473,7 +525,7 @@ export const DocsGPTWidget = ({
                       </MessageBubble>
                     }
                     {
-                      query.response ? <MessageBubble type='ANSWER'>
+                      query.response ? <MessageBubble onMouseOver={() => { isBubbleHovered.current = true }} type='ANSWER'>
                         <Message
                           type='ANSWER'
                           ref={(index === queries.length - 1) ? endMessageRef : null}
@@ -483,6 +535,24 @@ export const DocsGPTWidget = ({
                             dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(md.render(query.response)) }}
                           />
                         </Message>
+
+                        {collectFeedback &&
+                          <Feedback>
+                            <Like
+                              style={{
+                                stroke: query.feedback == 'LIKE' ? '#8860DB' : '#c0c0c0',
+                                visibility: query.feedback == 'LIKE' ? 'visible' : 'hidden'
+                              }}
+                              fill='none'
+                              onClick={() => handleFeedback("LIKE", index)} />
+                            <Dislike
+                              style={{
+                                stroke: query.feedback == 'DISLIKE' ? '#ed8085' : '#c0c0c0',
+                                visibility: query.feedback == 'DISLIKE' ? 'visible' : 'hidden'
+                              }}
+                              fill='none'
+                              onClick={() => handleFeedback("DISLIKE", index)} />
+                          </Feedback>}
                       </MessageBubble>
                         : <div>
                           {
@@ -518,7 +588,7 @@ export const DocsGPTWidget = ({
               type='text' placeholder="What do you want to do?" />
             <StyledButton
               size={size}
-              disabled={prompt.length == 0 || status !== 'idle'}>
+              disabled={prompt.trim().length == 0 || status !== 'idle'}>
               <PaperPlaneIcon width={15} height={15} color='white' />
             </StyledButton>
           </PromptContainer>

extensions/react-widget/src/requests/streamingApi.ts

@@ -1,3 +1,4 @@
+import { FEEDBACK } from "@/types";
 interface HistoryItem {
   prompt: string;
   response?: string;
@@ -11,6 +12,12 @@ interface FetchAnswerStreamingProps {
   apiHost?: string;
   onEvent?: (event: MessageEvent) => void;
 }
+interface FeedbackPayload {
+  question: string;
+  answer: string;
+  apikey: string;
+  feedback: FEEDBACK;
+}
 export function fetchAnswerStreaming({
   question = '',
   apiKey = '',
@@ -20,12 +27,12 @@ export function fetchAnswerStreaming({
   onEvent = () => { console.log("Event triggered, but no handler provided."); }
 }: FetchAnswerStreamingProps): Promise<void> {
   return new Promise<void>((resolve, reject) => {
-    const body= {
+    const body = {
       question: question,
       history: JSON.stringify(history),
       conversation_id: conversationId,
       model: 'default',
-      api_key:apiKey
+      api_key: apiKey
     };
     fetch(apiHost + '/stream', {
       method: 'POST',
@@ -80,4 +87,20 @@ export function fetchAnswerStreaming({
         reject(error);
       });
   });
-}
+}
+
+
+export const sendFeedback = (payload: FeedbackPayload,apiHost:string): Promise<Response> => {
+  return fetch(`${apiHost}/api/feedback`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      question: payload.question,
+      answer: payload.answer,
+      feedback: payload.feedback,
+      api_key:payload.apikey
+    }),
+  });
+};

extensions/react-widget/src/types/index.ts

@@ -23,4 +23,5 @@ export interface WidgetProps {
   theme?:THEME,
   buttonIcon?:string;
   buttonBg?:string;
+  collectFeedback?:boolean
 }

frontend/package.json

@@ -19,47 +19,50 @@
     ]
   },
   "dependencies": {
-    "@reduxjs/toolkit": "^1.9.2",
-    "@vercel/analytics": "^0.1.10",
-    "i18next": "^23.14.0",
+    "@reduxjs/toolkit": "^2.2.7",
+    "chart.js": "^4.4.4",
+    "i18next": "^23.15.1",
     "i18next-browser-languagedetector": "^8.0.0",
     "prop-types": "^15.8.1",
     "react": "^18.2.0",
+    "react-chartjs-2": "^5.2.0",
     "react-copy-to-clipboard": "^5.1.0",
     "react-dom": "^18.3.1",
     "react-dropzone": "^14.2.3",
-    "react-i18next": "^15.0.1",
+    "react-i18next": "^15.0.2",
     "react-markdown": "^9.0.1",
     "react-redux": "^8.0.5",
     "react-router-dom": "^6.8.1",
     "react-syntax-highlighter": "^15.5.0",
-    "remark-gfm": "^4.0.0"
+    "rehype-katex": "^7.0.1",
+    "remark-gfm": "^4.0.0",
+    "remark-math": "^6.0.0"
   },
   "devDependencies": {
     "@types/react": "^18.0.27",
     "@types/react-dom": "^18.3.0",
-    "@types/react-syntax-highlighter": "^15.5.6",
+    "@types/react-syntax-highlighter": "^15.5.13",
     "@typescript-eslint/eslint-plugin": "^5.51.0",
     "@typescript-eslint/parser": "^5.62.0",
     "@vitejs/plugin-react": "^4.3.1",
     "autoprefixer": "^10.4.13",
-    "eslint": "^8.33.0",
-    "eslint-config-prettier": "^8.6.0",
+    "eslint": "^8.57.1",
+    "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard-with-typescript": "^34.0.0",
-    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-import": "^2.30.0",
     "eslint-plugin-n": "^15.7.0",
     "eslint-plugin-prettier": "^5.2.1",
     "eslint-plugin-promise": "^6.6.0",
     "eslint-plugin-react": "^7.35.0",
-    "eslint-plugin-unused-imports": "^2.0.0",
+    "eslint-plugin-unused-imports": "^4.1.4",
     "husky": "^8.0.0",
-    "lint-staged": "^15.2.8",
+    "lint-staged": "^15.2.10",
     "postcss": "^8.4.41",
     "prettier": "^3.3.3",
-    "prettier-plugin-tailwindcss": "^0.2.2",
-    "tailwindcss": "^3.2.4",
-    "typescript": "^4.9.5",
-    "vite": "^5.3.5",
+    "prettier-plugin-tailwindcss": "^0.6.8",
+    "tailwindcss": "^3.4.11",
+    "typescript": "^5.6.2",
+    "vite": "^5.4.6",
     "vite-plugin-svgr": "^4.2.0"
   }
 }

frontend/src/App.tsx

@@ -3,7 +3,6 @@ import Navigation from './Navigation';
 import Conversation from './conversation/Conversation';
 import About from './About';
 import PageNotFound from './PageNotFound';
-import { inject } from '@vercel/analytics';
 import { useMediaQuery } from './hooks';
 import { useState } from 'react';
 import Setting from './settings';
@@ -11,16 +10,16 @@ import './locale/i18n';
 import { Outlet } from 'react-router-dom';
 import { SharedConversation } from './conversation/SharedConversation';
 import { useDarkTheme } from './hooks';
-inject();
 
 function MainLayout() {
   const { isMobile } = useMediaQuery();
   const [navOpen, setNavOpen] = useState(!isMobile);
+
   return (
-    <div className="dark:bg-raisin-black">
+    <div className="dark:bg-raisin-black relative h-screen overflow-auto">
       <Navigation navOpen={navOpen} setNavOpen={setNavOpen} />
       <div
-        className={`min-h-screen ${
+        className={`h-[calc(100dvh-64px)] md:h-screen ${
           !isMobile
             ? `ml-0 ${!navOpen ? 'md:mx-auto lg:mx-auto' : 'md:ml-72'}`
             : 'ml-0 md:ml-16'
@@ -33,9 +32,12 @@ function MainLayout() {
 }
 
 export default function App() {
-  useDarkTheme();
+  const [, , componentMounted] = useDarkTheme();
+  if (!componentMounted) {
+    return <div />;
+  }
   return (
-    <>
+    <div className="h-full relative overflow-auto">
       <Routes>
         <Route element={<MainLayout />}>
           <Route index element={<Conversation />} />
@@ -45,6 +47,6 @@ export default function App() {
         <Route path="/share/:identifier" element={<SharedConversation />} />
         <Route path="/*" element={<PageNotFound />} />
       </Routes>
-    </>
+    </div>
   );
 }

frontend/src/Hero.tsx

@@ -19,7 +19,7 @@ export default function Hero({
   }>;
   return (
     <div
-      className={`mb-1 mt-16 flex w-full flex-col justify-end text-black-1000 dark:text-bright-gray sm:w-full md:mb-10 lg:mt-6`}
+      className={`pt-20 sm:pt-0 pb-6 sm:pb-12 flex h-full  w-full flex-col  text-black-1000 dark:text-bright-gray sm:w-full px-2 sm:px-0`}
     >
       <div className="flex h-full w-full flex-col items-center justify-center">
         <div className="flex items-center">
@@ -37,7 +37,7 @@ export default function Hero({
               <Fragment key={key}>
                 <button
                   onClick={() => handleQuestion({ question: demo.query })}
-                  className="w-full rounded-full border-2 border-silver px-6 py-4 text-left hover:border-gray-4000 dark:hover:border-gray-3000 xl:min-w-[24vw]"
+                  className="w-full rounded-full border border-silver px-6 py-4 text-left hover:border-gray-4000 dark:hover:border-gray-3000 xl:min-w-[24vw]"
                 >
                   <p className="mb-1 font-semibold text-black dark:text-silver">
                     {demo.header}

frontend/src/Navigation.tsx

@@ -2,7 +2,6 @@ import { useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
 import { NavLink, useNavigate } from 'react-router-dom';
-
 import conversationService from './api/services/conversationService';
 import userService from './api/services/userService';
 import Add from './assets/add.svg';
@@ -10,8 +9,6 @@ import DocsGPT3 from './assets/cute_docsgpt3.svg';
 import Discord from './assets/discord.svg';
 import Expand from './assets/expand.svg';
 import Github from './assets/github.svg';
-import Hamburger from './assets/hamburger.svg';
-import Info from './assets/info.svg';
 import SettingGear from './assets/settingGear.svg';
 import Twitter from './assets/TwitterX.svg';
 import UploadIcon from './assets/upload.svg';
@@ -24,9 +21,9 @@ import ConversationTile from './conversation/ConversationTile';
 import { useDarkTheme, useMediaQuery, useOutsideAlerter } from './hooks';
 import useDefaultDocument from './hooks/useDefaultDocument';
 import DeleteConvModal from './modals/DeleteConvModal';
-import { ActiveState } from './models/misc';
+import { ActiveState, Doc } from './models/misc';
 import APIKeyModal from './preferences/APIKeyModal';
-import { Doc, getConversations, getDocs } from './preferences/preferenceApi';
+import { getConversations, getDocs } from './preferences/preferenceApi';
 import {
   selectApiKeyStatus,
   selectConversationId,
@@ -41,6 +38,9 @@ import {
   setSourceDocs,
 } from './preferences/preferenceSlice';
 import Upload from './upload/Upload';
+import ShareButton from './components/ShareButton';
+import Help from './components/Help';
+
 
 interface NavigationProps {
   navOpen: boolean;
@@ -73,7 +73,6 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   const [isDarkTheme] = useDarkTheme();
   const [isDocsListOpen, setIsDocsListOpen] = useState(false);
   const { t } = useTranslation();
-
   const isApiKeySet = useSelector(selectApiKeyStatus);
   const [apiKeyModalState, setApiKeyModalState] =
     useState<ActiveState>('INACTIVE');
@@ -119,15 +118,14 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
       .delete(id, {})
       .then(() => {
         fetchConversations();
+        resetConversation();
       })
       .catch((error) => console.error(error));
   };
 
   const handleDeleteClick = (doc: Doc) => {
-    const docPath = `indexes/local/${doc.name}`;
-
     userService
-      .deletePath(docPath)
+      .deletePath(doc.id ?? '')
       .then(() => {
         return getDocs();
       })
@@ -157,6 +155,15 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
       });
   };
 
+  const resetConversation = () => {
+    dispatch(setConversation([]));
+    dispatch(
+      updateConversationId({
+        query: { conversationId: null },
+      }),
+    );
+  };
+
   async function updateConversationName(updatedConversation: {
     name: string;
     id: string;
@@ -216,9 +223,18 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         <div
           className={'visible mt-2 flex h-[6vh] w-full justify-between md:h-12'}
         >
-          <div className="my-auto mx-4 flex cursor-pointer gap-1.5">
-            <img className="mb-2 h-10" src={DocsGPT3} alt="" />
-            <p className="my-auto text-2xl font-semibold">DocsGPT</p>
+          <div
+            className="my-auto mx-4 flex cursor-pointer gap-1.5"
+            onClick={() => {
+              if (isMobile) {
+                setNavOpen(!navOpen);
+              }
+            }}
+          >
+            <a href="/" className="flex gap-1.5">
+              <img className="mb-2 h-10" src={DocsGPT3} alt="" />
+              <p className="my-auto text-2xl font-semibold">DocsGPT</p>
+            </a>
           </div>
           <button
             className="float-right mr-5"
@@ -238,12 +254,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         <NavLink
           to={'/'}
           onClick={() => {
-            dispatch(setConversation([]));
-            dispatch(
-              updateConversationId({
-                query: { conversationId: null },
-              }),
-            );
+            if (isMobile) {
+              setNavOpen(!navOpen);
+            }
+            resetConversation();
           }}
           className={({ isActive }) =>
             `${
@@ -260,7 +274,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             {t('newChat')}
           </p>
         </NavLink>
-        <div className="mb-auto h-[78vh] overflow-y-auto overflow-x-hidden dark:text-white">
+        <div
+          id="conversationsMainDiv"
+          className="mb-auto h-[78vh] overflow-y-auto overflow-x-hidden dark:text-white"
+        >
           {conversations && conversations.length > 0 ? (
             <div>
               <div className=" my-auto mx-4 mt-2 flex h-6 items-center justify-between gap-4 rounded-3xl">
@@ -272,6 +289,11 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                     key={conversation.id}
                     conversation={conversation}
                     selectConversation={(id) => handleConversationClick(id)}
+                    onCoversationClick={() => {
+                      if (isMobile) {
+                        setNavOpen(false);
+                      }
+                    }}
                     onDeleteConversation={(id) => handleDeleteConversation(id)}
                     onSave={(conversation) =>
                       updateConversationName(conversation)
@@ -284,7 +306,6 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             <></>
           )}
         </div>
-
         <div className="flex h-auto flex-col justify-end text-eerie-black dark:text-white">
           <div className="flex flex-col-reverse border-b-[1px] dark:border-b-purple-taupe">
             <div className="relative my-4 mx-4 flex gap-2">
@@ -295,17 +316,33 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                 isDocsListOpen={isDocsListOpen}
                 setIsDocsListOpen={setIsDocsListOpen}
                 handleDeleteClick={handleDeleteClick}
+                handlePostDocumentSelect={(option?: string) => {
+                  if (isMobile) {
+                    setNavOpen(!navOpen);
+                  }
+                }}
               />
               <img
                 className="mt-2 h-9 w-9 hover:cursor-pointer"
                 src={UploadIcon}
-                onClick={() => setUploadModalState('ACTIVE')}
+                onClick={() => {
+                  setUploadModalState('ACTIVE');
+                  if (isMobile) {
+                    setNavOpen(!navOpen);
+                  }
+                }}
               ></img>
             </div>
             <p className="ml-5 mt-3 text-sm font-semibold">{t('sourceDocs')}</p>
           </div>
           <div className="flex flex-col gap-2 border-b-[1px] py-2 dark:border-b-purple-taupe">
             <NavLink
+              onClick={() => {
+                if (isMobile) {
+                  setNavOpen(!navOpen);
+                }
+                resetConversation();
+              }}
               to="/settings"
               className={({ isActive }) =>
                 `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
@@ -323,77 +360,72 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
               </p>
             </NavLink>
           </div>
-          <div className="flex justify-between gap-2 border-b-[1.5px] py-2 dark:border-b-purple-taupe">
-            <NavLink
-              to="/about"
-              className={({ isActive }) =>
-                `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
-                  isActive ? 'bg-gray-3000 dark:bg-[#28292E]' : ''
-                }`
-              }
-            >
-              <img
-                src={Info}
-                alt="icon"
-                className="ml-2 w-5 filter dark:invert"
-              />
-              <p className="my-auto pr-1 text-sm">{t('about')}</p>
-            </NavLink>
-            <div className="flex items-center justify-evenly gap-1 px-1">
-              <NavLink
-                target="_blank"
-                to={'https://discord.gg/WHJdfbQDR4'}
-                className={
-                  'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                }
-              >
-                <img
-                  src={Discord}
-                  alt="discord"
-                  className="m-2 w-6 self-center filter dark:invert"
-                />
-              </NavLink>
-              <NavLink
-                target="_blank"
-                to={'https://twitter.com/docsgptai'}
-                className={
-                  'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                }
-              >
-                <img
-                  src={Twitter}
-                  alt="x"
-                  className="m-2 w-5 self-center filter dark:invert"
-                />
-              </NavLink>
-              <NavLink
-                target="_blank"
-                to={'https://github.com/arc53/docsgpt'}
-                className={
-                  'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                }
-              >
-                <img
-                  src={Github}
-                  alt="github"
-                  className="m-2 w-6 self-center filter dark:invert"
-                />
-              </NavLink>
+          <div className="flex flex-col justify-end text-eerie-black dark:text-white">
+            <div className="flex justify-between items-center px-1 py-1">
+              <Help />
+
+              <div className="flex items-center gap-1">
+                <NavLink
+                  target="_blank"
+                  to={'https://discord.gg/WHJdfbQDR4'}
+                  className={
+                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
+                  }
+                >
+                  <img
+                    src={Discord}
+                    alt="discord"
+                    className="m-2 w-6 self-center filter dark:invert"
+                  />
+                </NavLink>
+                <NavLink
+                  target="_blank"
+                  to={'https://twitter.com/docsgptai'}
+                  className={
+                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
+                  }
+                >
+                  <img
+                    src={Twitter}
+                    alt="x"
+                    className="m-2 w-5 self-center filter dark:invert"
+                  />
+                </NavLink>
+                <NavLink
+                  target="_blank"
+                  to={'https://github.com/arc53/docsgpt'}
+                  className={
+                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
+                  }
+                >
+                  <img
+                    src={Github}
+                    alt="github"
+                    className="m-2 w-6 self-center filter dark:invert"
+                  />
+                </NavLink>
+              </div>
             </div>
           </div>
         </div>
       </div>
-      <div className="fixed z-10 h-16 w-full border-b-2 bg-gray-50 dark:border-b-purple-taupe dark:bg-chinese-black md:hidden">
+      <div className="sticky z-10 h-16 w-full border-b-2 bg-gray-50 dark:border-b-purple-taupe dark:bg-chinese-black md:hidden">
         <button
-          className="mt-5 ml-6 h-6 w-6 md:hidden"
-          onClick={() => setNavOpen(true)}
+          className="m-5"
+          onClick={() => {
+            setNavOpen(!navOpen);
+          }}
         >
           <img
-            src={Hamburger}
+            src={Expand}
             alt="menu toggle"
-            className="w-7 filter dark:invert"
+            className={`${
+              !navOpen ? 'rotate-180' : 'rotate-0'
+            } m-auto transition-all duration-200`}
           />
         </button>
+
+        {conversationId && <ShareButton conversationId={conversationId} />}
       </div>
       <APIKeyModal
         modalState={apiKeyModalState}
@@ -408,6 +440,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
       <Upload
         modalState={uploadModalState}
         setModalState={setUploadModalState}
+        isOnboarding={false}
       ></Upload>
     </>
   );

frontend/src/api/endpoints.ts

@@ -10,8 +10,13 @@ const endpoints = {
     DELETE_PROMPT: '/api/delete_prompt',
     UPDATE_PROMPT: '/api/update_prompt',
     SINGLE_PROMPT: (id: string) => `/api/get_single_prompt?id=${id}`,
-    DELETE_PATH: (docPath: string) => `/api/delete_old?path=${docPath}`,
+    DELETE_PATH: (docPath: string) => `/api/delete_old?source_id=${docPath}`,
     TASK_STATUS: (task_id: string) => `/api/task_status?task_id=${task_id}`,
+    MESSAGE_ANALYTICS: '/api/get_message_analytics',
+    TOKEN_ANALYTICS: '/api/get_token_analytics',
+    FEEDBACK_ANALYTICS: '/api/get_feedback_analytics',
+    LOGS: `/api/get_user_logs`,
+    MANAGE_SYNC: '/api/manage_sync',
   },
   CONVERSATION: {
     ANSWER: '/api/answer',

frontend/src/api/services/userService.ts

@@ -23,6 +23,16 @@ const userService = {
     apiClient.get(endpoints.USER.DELETE_PATH(docPath)),
   getTaskStatus: (task_id: string): Promise<any> =>
     apiClient.get(endpoints.USER.TASK_STATUS(task_id)),
+  getMessageAnalytics: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.MESSAGE_ANALYTICS, data),
+  getTokenAnalytics: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.TOKEN_ANALYTICS, data),
+  getFeedbackAnalytics: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.FEEDBACK_ANALYTICS, data),
+  getLogs: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.LOGS, data),
+  manageSync: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.MANAGE_SYNC, data),
 };
 
 export default userService;

frontend/src/assets/chevron-right.svg

@@ -0,0 +1,3 @@
+<svg width="7" height="12" viewBox="0 0 7 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.29154 4.88202L1.70154 0.29202C1.60896 0.199438 1.49905 0.125998 1.37808 0.0758932C1.25712 0.0257882 1.12747 -2.37536e-07 0.99654 -2.44235e-07C0.86561 -2.50934e-07 0.735961 0.0257882 0.614997 0.0758931C0.494033 0.125998 0.384122 0.199438 0.29154 0.29202C0.198958 0.384602 0.125519 0.494513 0.0754137 0.615477C0.0253086 0.736441 -0.00048069 0.86609 -0.000480695 0.99702C-0.000480701 1.12795 0.0253086 1.2576 0.0754136 1.37856C0.125519 1.49953 0.198958 1.60944 0.29154 1.70202L4.17154 5.59202L0.29154 9.47202C0.198958 9.5646 0.125518 9.67451 0.0754133 9.79548C0.0253082 9.91644 -0.000481091 10.0461 -0.000481097 10.177C-0.000481102 10.3079 0.0253082 10.4376 0.0754132 10.5586C0.125518 10.6795 0.198958 10.7894 0.29154 10.882C0.384121 10.9746 0.494032 11.048 0.614996 11.0981C0.73596 11.1483 0.865609 11.174 0.99654 11.174C1.12747 11.174 1.25712 11.1483 1.37808 11.0981C1.49905 11.048 1.60896 10.9746 1.70154 10.882L6.29154 6.29202C6.38424 6.19951 6.45779 6.08962 6.50797 5.96864C6.55815 5.84767 6.58398 5.71799 6.58398 5.58702C6.58398 5.45605 6.55815 5.32637 6.50797 5.2054C6.45779 5.08442 6.38424 4.97453 6.29154 4.88202Z" fill="#666666"/>
+</svg>

frontend/src/assets/documentation-dark.svg

@@ -1,3 +1,4 @@
-<svg xmlns="http://www.w3.org/2000/svg" x="0px" y="0px" width="100" height="100" viewBox="0,0,256,256">
-    <g transform="translate(-19.2,-19.2) scale(1.15,1.15)"><g fill="#949494" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M13.172,2h-7.172c-1.1,0 -2,0.9 -2,2v16c0,1.1 0.9,2 2,2h12c1.1,0 2,-0.9 2,-2v-11.172c0,-0.53 -0.211,-1.039 -0.586,-1.414l-4.828,-4.828c-0.375,-0.375 -0.884,-0.586 -1.414,-0.586zM15,18h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM15,14h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM13,9v-5.5l5.5,5.5z"></path></g></g></g>
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16.25 8.64258V16.25C16.25 16.7473 16.0525 17.2242 15.7008 17.5758C15.3492 17.9275 14.8723 18.125 14.375 18.125H5.625C5.12772 18.125 4.65081 17.9275 4.29917 17.5758C3.94754 17.2242 3.75 16.7473 3.75 16.25V3.75C3.75 3.25272 3.94754 2.77581 4.29917 2.42417C4.65081 2.07254 5.12772 1.875 5.625 1.875H9.48242C9.81383 1.87505 10.1316 2.0067 10.366 2.24102L15.884 7.75898C16.1183 7.99335 16.2499 8.31117 16.25 8.64258Z" stroke="#ECECF1" stroke-width="1.11111" stroke-linejoin="round"/>
+<path d="M10 2.1875V6.875C10 7.20652 10.1317 7.52446 10.3661 7.75888C10.6005 7.9933 10.9185 8.125 11.25 8.125H15.9375M6.875 11.25H13.125M6.875 14.375H13.125" stroke="#ECECF1" stroke-width="1.11111" stroke-linecap="round" stroke-linejoin="round"/>
 </svg>

frontend/src/assets/documentation.svg

@@ -1,3 +1,4 @@
-<svg xmlns="http://www.w3.org/2000/svg" x="0px" y="0px" width="100" height="100" viewBox="0,0,256,256">
-    <g transform="translate(-19.2,-19.2) scale(1.15,1.15)"><g fill="black" fill-opacity="0.54" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M13.172,2h-7.172c-1.1,0 -2,0.9 -2,2v16c0,1.1 0.9,2 2,2h12c1.1,0 2,-0.9 2,-2v-11.172c0,-0.53 -0.211,-1.039 -0.586,-1.414l-4.828,-4.828c-0.375,-0.375 -0.884,-0.586 -1.414,-0.586zM15,18h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM15,14h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM13,9v-5.5l5.5,5.5z"></path></g></g></g>
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16.25 8.64258V16.25C16.25 16.7473 16.0525 17.2242 15.7008 17.5758C15.3492 17.9275 14.8723 18.125 14.375 18.125H5.625C5.12772 18.125 4.65081 17.9275 4.29917 17.5758C3.94754 17.2242 3.75 16.7473 3.75 16.25V3.75C3.75 3.25272 3.94754 2.77581 4.29917 2.42417C4.65081 2.07254 5.12772 1.875 5.625 1.875H9.48242C9.81383 1.87505 10.1316 2.0067 10.366 2.24102L15.884 7.75898C16.1183 7.99335 16.2499 8.31117 16.25 8.64258Z" stroke="#6E6E6E" stroke-width="1.11111" stroke-linejoin="round"/>
+<path d="M10 2.1875V6.875C10 7.20652 10.1317 7.52446 10.3661 7.75888C10.6005 7.9933 10.9185 8.125 11.25 8.125H15.9375M6.875 11.25H13.125M6.875 14.375H13.125" stroke="#6E6E6E" stroke-width="1.11111" stroke-linecap="round" stroke-linejoin="round"/>
 </svg>

frontend/src/assets/envelope-dark.svg

@@ -0,0 +1,3 @@
+<svg width="18" height="18" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M3.04867 15.49H15.1775C16.5339 15.49 17.3172 14.7067 17.3172 13.1548V4.83755C17.3172 3.29309 16.5265 2.50977 14.9515 2.50977H2.82302C1.47463 2.50977 0.683594 3.28569 0.683594 4.83755V13.1545C0.683594 14.7138 1.48138 15.49 3.04867 15.49ZM8.10441 9.3803L2.41609 3.76816C2.58163 3.70034 2.77738 3.66273 3.01106 3.66273H14.9971C15.2305 3.66273 15.434 3.70034 15.6072 3.78327L9.92692 9.38062C9.60291 9.7043 9.31652 9.84766 9.01534 9.84766C8.71384 9.84766 8.42841 9.70398 8.10441 9.3803ZM1.83559 13.1548V4.76234L6.16717 9.01162L1.84331 13.2824C1.83592 13.2448 1.83559 13.1998 1.83559 13.1548ZM16.1646 4.84494V13.2599L11.8629 9.0113L16.1649 4.78516L16.1646 4.84494ZM3.01106 14.3377C2.79249 14.3377 2.61184 14.3075 2.4537 14.2397L6.95852 9.78723L7.44838 10.2694C7.97552 10.7891 8.48017 11.0077 9.01534 11.0077C9.54249 11.0077 10.0548 10.7891 10.5823 10.2694L11.0718 9.78723L15.5696 14.2319C15.4111 14.3075 15.2154 14.3374 14.9968 14.3374L3.01106 14.3377Z" fill="#ECECF1"/>
+</svg>

frontend/src/assets/envelope.svg

@@ -0,0 +1,3 @@
+<svg width="18" height="18" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M3.04867 15.49H15.1775C16.5339 15.49 17.3172 14.7067 17.3172 13.1548V4.83755C17.3172 3.29309 16.5265 2.50977 14.9515 2.50977H2.82302C1.47463 2.50977 0.683594 3.28569 0.683594 4.83755V13.1545C0.683594 14.7138 1.48138 15.49 3.04867 15.49ZM8.10441 9.3803L2.41609 3.76816C2.58163 3.70034 2.77738 3.66273 3.01106 3.66273H14.9971C15.2305 3.66273 15.434 3.70034 15.6072 3.78327L9.92692 9.38062C9.60291 9.7043 9.31652 9.84766 9.01534 9.84766C8.71384 9.84766 8.42841 9.70398 8.10441 9.3803ZM1.83559 13.1548V4.76234L6.16717 9.01162L1.84331 13.2824C1.83592 13.2448 1.83559 13.1998 1.83559 13.1548ZM16.1646 4.84494V13.2599L11.8629 9.0113L16.1649 4.78516L16.1646 4.84494ZM3.01106 14.3377C2.79249 14.3377 2.61184 14.3075 2.4537 14.2397L6.95852 9.78723L7.44838 10.2694C7.97552 10.7891 8.48017 11.0077 9.01534 11.0077C9.54249 11.0077 10.0548 10.7891 10.5823 10.2694L11.0718 9.78723L15.5696 14.2319C15.4111 14.3075 15.2154 14.3374 14.9968 14.3374L3.01106 14.3377Z" fill="#6F6F6F"/>
+</svg>

frontend/src/assets/file_upload.svg

@@ -0,0 +1,3 @@
+<svg width="28" height="34" viewBox="0 0 28 34" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M10 26.0003H18C19.1 26.0003 20 25.1003 20 24.0003V14.0003H23.18C24.96 14.0003 25.86 11.8403 24.6 10.5803L15.42 1.40032C15.235 1.21491 15.0152 1.06782 14.7732 0.967453C14.5313 0.86709 14.2719 0.81543 14.01 0.81543C13.7481 0.81543 13.4887 0.86709 13.2468 0.967453C13.0048 1.06782 12.785 1.21491 12.6 1.40032L3.42 10.5803C2.16 11.8403 3.04 14.0003 4.82 14.0003H8V24.0003C8 25.1003 8.9 26.0003 10 26.0003ZM2 30.0003H26C27.1 30.0003 28 30.9003 28 32.0003C28 33.1003 27.1 34.0003 26 34.0003H2C0.9 34.0003 0 33.1003 0 32.0003C0 30.9003 0.9 30.0003 2 30.0003Z" fill="#828282"/>
+</svg>

frontend/src/assets/sync.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="24px" viewBox="0 0 24 24" width="24px" fill="#ffffff"><path d="M.01 0h24v24h-24V0z" fill="none"/><path d="M12 4V1L8 5l4 4V6c3.31 0 6 2.69 6 6 0 1.01-.25 1.97-.7 2.8l1.46 1.46C19.54 15.03 20 13.57 20 12c0-4.42-3.58-8-8-8zm0 14c-3.31 0-6-2.69-6-6 0-1.01.25-1.97.7-2.8L5.24 7.74C4.46 8.97 4 10.43 4 12c0 4.42 3.58 8 8 8v3l4-4-4-4v3z"/></svg>

frontend/src/assets/website_collect.svg

@@ -0,0 +1,6 @@
+<svg width="58" height="49" viewBox="0 0 58 49" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M33.0002 45.3337H6.3335C5.27263 45.3337 4.25521 44.9122 3.50507 44.1621C2.75492 43.4119 2.3335 42.3945 2.3335 41.3337V6.66699C2.3335 5.60613 2.75492 4.58871 3.50507 3.83856C4.25521 3.08842 5.27263 2.66699 6.3335 2.66699H51.6668C52.7277 2.66699 53.7451 3.08842 54.4953 3.83856C55.2454 4.58871 55.6668 5.60613 55.6668 6.66699V24.0003M42.3335 40.0003L49.0002 46.667M49.0002 46.667L55.6668 40.0003M49.0002 46.667V30.667" stroke="#626262" stroke-width="4" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M2.3335 6.66699C2.3335 5.60613 2.75492 4.58871 3.50507 3.83856C4.25521 3.08842 5.27263 2.66699 6.3335 2.66699H51.6668C52.7277 2.66699 53.7451 3.08842 54.4953 3.83856C55.2454 4.58871 55.6668 5.60613 55.6668 6.66699V18.667H2.3335V6.66699Z" stroke="#626262" stroke-width="4"/>
+<path d="M7.66667 10.6673C7.66667 9.19456 8.86057 8.00065 10.3333 8.00065C11.8061 8.00065 13 9.19456 13 10.6673C13 12.1401 11.8061 13.334 10.3333 13.334C8.86057 13.334 7.66667 12.1401 7.66667 10.6673Z" fill="#626262"/>
+<path d="M15.6667 10.6673C15.6667 9.19456 16.8606 8.00065 18.3333 8.00065C19.8061 8.00065 21 9.19456 21 10.6673C21 12.1401 19.8061 13.334 18.3333 13.334C16.8606 13.334 15.6667 12.1401 15.6667 10.6673Z" fill="#626262"/>
+</svg>

frontend/src/components/CopyButton.tsx

@@ -1,16 +1,24 @@
-import { useState } from 'react';
-import Copy from './../assets/copy.svg?react';
-import CheckMark from './../assets/checkmark.svg?react';
 import copy from 'copy-to-clipboard';
+import { useState } from 'react';
 
-export default function CoppyButton({ text }: { text: string }) {
+import CheckMark from '../assets/checkmark.svg?react';
+import Copy from '../assets/copy.svg?react';
+
+export default function CoppyButton({
+  text,
+  colorLight,
+  colorDark,
+}: {
+  text: string;
+  colorLight?: string;
+  colorDark?: string;
+}) {
   const [copied, setCopied] = useState(false);
   const [isCopyHovered, setIsCopyHovered] = useState(false);
 
   const handleCopyClick = (text: string) => {
     copy(text);
     setCopied(true);
-    // Reset copied to false after a few seconds
     setTimeout(() => {
       setCopied(false);
     }, 3000);
@@ -20,8 +28,8 @@ export default function CoppyButton({ text }: { text: string }) {
     <div
       className={`flex items-center justify-center rounded-full p-2 ${
         isCopyHovered
-          ? 'bg-[#EEEEEE] dark:bg-purple-taupe'
-          : 'bg-[#ffffff] dark:bg-transparent'
+          ? `bg-[#EEEEEE] dark:bg-purple-taupe`
+          : `bg-[${colorLight ? colorLight : '#FFFFFF'}] dark:bg-[${colorDark ? colorDark : 'transparent'}]`
       }`}
     >
       {copied ? (

frontend/src/components/Dropdown.tsx

@@ -16,6 +16,7 @@ function Dropdown({
   showDelete,
   onDelete,
   placeholder,
+  contentSize = 'text-base',
 }: {
   options:
     | string[]
@@ -26,6 +27,7 @@ function Dropdown({
     | string
     | { label: string; value: string }
     | { value: number; description: string }
+    | { name: string; id: string; type: string }
     | null;
   onSelect:
     | ((value: string) => void)
@@ -41,6 +43,7 @@ function Dropdown({
   showDelete?: boolean;
   onDelete?: (value: string) => void;
   placeholder?: string;
+  contentSize?: string;
 }) {
   const dropdownRef = React.useRef<HTMLDivElement>(null);
   const [isOpen, setIsOpen] = React.useState(false);
@@ -79,14 +82,14 @@ function Dropdown({
         }`}
       >
         {typeof selectedValue === 'string' ? (
-          <span className="overflow-hidden text-ellipsis dark:text-bright-gray">
+          <span className="truncate dark:text-bright-gray">
             {selectedValue}
           </span>
         ) : (
           <span
-            className={`overflow-hidden text-ellipsis dark:text-bright-gray ${
+            className={`truncate dark:text-bright-gray ${
               !selectedValue && 'text-silver dark:text-gray-400'
-            }`}
+            } ${contentSize}`}
           >
             {selectedValue && 'label' in selectedValue
               ? selectedValue.label
@@ -123,7 +126,7 @@ function Dropdown({
                   onSelect(option);
                   setIsOpen(false);
                 }}
-                className="ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3 dark:text-light-gray"
+                className={`ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3 dark:text-light-gray ${contentSize}`}
               >
                 {typeof option === 'string'
                   ? option

frontend/src/components/DropdownMenu.tsx

@@ -0,0 +1,88 @@
+import React from 'react';
+
+type DropdownMenuProps = {
+  name: string;
+  options: { label: string; value: string }[];
+  onSelect: (value: string) => void;
+  defaultValue?: string;
+  icon?: string;
+};
+
+export default function DropdownMenu({
+  name,
+  options,
+  onSelect,
+  defaultValue = 'none',
+  icon,
+}: DropdownMenuProps) {
+  const dropdownRef = React.useRef<HTMLDivElement>(null);
+  const [isOpen, setIsOpen] = React.useState(false);
+  const [selectedOption, setSelectedOption] = React.useState(
+    options.find((option) => option.value === defaultValue) || options[0],
+  );
+
+  const handleToggle = () => {
+    setIsOpen(!isOpen);
+  };
+  const handleClickOutside = (event: MouseEvent) => {
+    if (
+      dropdownRef.current &&
+      !dropdownRef.current.contains(event.target as Node)
+    ) {
+      setIsOpen(false);
+    }
+  };
+  const handleClickOption = (optionId: number) => {
+    setIsOpen(false);
+    setSelectedOption(options[optionId]);
+    onSelect(options[optionId].value);
+  };
+
+  React.useEffect(() => {
+    document.addEventListener('mousedown', handleClickOutside);
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+    };
+  }, []);
+  return (
+    <div className="static inline-block text-left" ref={dropdownRef}>
+      <button
+        onClick={handleToggle}
+        className="flex w-20 cursor-pointer flex-row items-center gap-px rounded-3xl border-purple-30/25 bg-purple-30 p-2 text-xs text-white hover:bg-[#6F3FD1] focus:outline-none"
+      >
+        {icon && <img src={icon} alt="OptionIcon" className="h-4 w-4" />}
+        {selectedOption.value !== 'never' ? selectedOption.label : name}
+      </button>
+      <div
+        className={`absolute z-50 right-0 mt-1 w-28 transform rounded-md bg-transparent shadow-lg ring-1 ring-black ring-opacity-5 transition-all duration-200 ease-in-out ${
+          isOpen
+            ? 'scale-100 opacity-100'
+            : 'pointer-events-none scale-95 opacity-0'
+        }`}
+      >
+        <div
+          role="menu"
+          className="overflow-hidden rounded-md"
+          aria-orientation="vertical"
+          aria-labelledby="options-menu"
+        >
+          {options.map((option, idx) => (
+            <div
+              id={`option-${idx}`}
+              className={`cursor-pointer px-4 py-2 text-xs hover:bg-gray-100 dark:text-light-gray dark:hover:bg-purple-taupe ${
+                selectedOption.value === option.value
+                  ? 'bg-gray-100 dark:bg-purple-taupe'
+                  : 'bg-white dark:bg-dark-charcoal'
+              }`}
+              role="menuitem"
+              key={option.value}
+              onClick={() => handleClickOption(idx)}
+            >
+              {option.label}
+            </div>
+          ))}
+        </div>
+      </div>
+    </div>
+  );
+}

frontend/src/components/Help.tsx

@@ -0,0 +1,80 @@
+import { useState, useRef, useEffect } from 'react';
+import Info from '../assets/info.svg';
+import PageIcon from '../assets/documentation.svg';
+import EmailIcon from '../assets/envelope.svg';
+import { useTranslation } from 'react-i18next';
+const Help = () => {
+  const [isOpen, setIsOpen] = useState(false);
+  const dropdownRef = useRef<HTMLDivElement | null>(null);
+  const buttonRef = useRef<HTMLButtonElement | null>(null);
+  const { t } = useTranslation();
+
+  const toggleDropdown = () => {
+    setIsOpen((prev) => !prev);
+  };
+
+  const handleClickOutside = (event: MouseEvent) => {
+    if (
+      dropdownRef.current &&
+      !dropdownRef.current.contains(event.target as Node) &&
+      buttonRef.current &&
+      !buttonRef.current.contains(event.target as Node)
+    ) {
+      setIsOpen(false);
+    }
+  };
+
+  useEffect(() => {
+    document.addEventListener('mousedown', handleClickOutside);
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+    };
+  }, []);
+
+  return (
+    <div className="relative inline-block text-sm" ref={dropdownRef}>
+      <button
+        ref={buttonRef}
+        onClick={toggleDropdown}
+        className="my-auto mx-4 w-full flex items-center h-9 gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E]"
+      >
+        <img src={Info} alt="info" className="ml-1 w-5 filter dark:invert" />
+        {t('help')}
+      </button>
+      {isOpen && (
+        <div
+          className={`absolute translate-x-4 -translate-y-28 z-10 w-48 shadow-lg bg-white dark:bg-[#444654] rounded-xl`}
+        >
+          <a
+            href="https://docs.docsgpt.cloud/"
+            target="_blank"
+            rel="noopener noreferrer"
+            className="flex items-start gap-4 px-4 py-2 text-black dark:text-white hover:bg-bright-gray dark:hover:bg-[#545561] rounded-t-xl"
+          >
+            <img
+              src={PageIcon}
+              alt="Documentation"
+              className="filter dark:invert"
+              width={20}
+            />
+            {t('documentation')}
+          </a>
+          <a
+            href="mailto:contact@arc53.com"
+            className="flex items-start gap-4 px-4 py-2 text-black dark:text-white hover:bg-bright-gray dark:hover:bg-[#545561] rounded-b-xl"
+          >
+            <img
+              src={EmailIcon}
+              alt="Email Us"
+              className="filter dark:invert p-0.5"
+              width={20}
+            />
+            {t('emailUs')}
+          </a>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default Help;

frontend/src/components/Input.tsx

@@ -10,6 +10,7 @@ const Input = ({
   maxLength,
   className,
   colorVariant = 'silver',
+  borderVariant = 'thick',
   children,
   onChange,
   onPaste,
@@ -20,10 +21,13 @@ const Input = ({
     jet: 'border-jet',
     gray: 'border-gray-5000 dark:text-silver',
   };
-
+  const borderStyles = {
+    thin: 'border',
+    thick: 'border-2',
+  };
   return (
     <input
-      className={`h-[42px] w-full rounded-full border-2 px-3 outline-none dark:bg-transparent dark:text-white ${className} ${colorStyles[colorVariant]}`}
+      className={`h-[42px] w-full rounded-full px-3 py-1 outline-none dark:bg-transparent dark:text-white ${className} ${colorStyles[colorVariant]} ${borderStyles[borderVariant]}`}
       type={type}
       id={id}
       name={name}

frontend/src/components/RetryIcon.tsx

@@ -4,10 +4,11 @@ const RetryIcon = (props: SVGProps<SVGSVGElement>) => (
   <svg
     xmlns="http://www.w3.org/2000/svg"
     xmlSpace="preserve"
-    width={16}
-    height={16}
+    width={props.width ? props.width : 16}
+    height={props.height ? props.height : 16}
     fill={props.fill}
-    stroke={props.stroke}
+    stroke={props.stroke ? props.stroke : 'none'}
+    strokeWidth={props.strokeWidth ? props.strokeWidth : 10}
     viewBox="0 0 383.748 383.748"
     {...props}
   >

frontend/src/components/SettingsBar.tsx

@@ -0,0 +1,96 @@
+import React, { useCallback, useRef, useState } from 'react';
+import ArrowLeft from '../assets/arrow-left.svg';
+import ArrowRight from '../assets/arrow-right.svg';
+import { useTranslation } from 'react-i18next';
+
+type HiddenGradientType = 'left' | 'right' | undefined;
+
+const useTabs = () => {
+  const { t } = useTranslation();
+  const tabs = [
+    t('settings.general.label'),
+    t('settings.documents.label'),
+    t('settings.apiKeys.label'),
+    t('settings.analytics.label'),
+    t('settings.logs.label'),
+  ];
+  return tabs;
+};
+
+interface SettingsBarProps {
+  setActiveTab: React.Dispatch<React.SetStateAction<string>>;
+  activeTab: string;
+}
+
+const SettingsBar = ({ setActiveTab, activeTab }: SettingsBarProps) => {
+  const [hiddenGradient, setHiddenGradient] =
+    useState<HiddenGradientType>('left');
+  const containerRef = useRef<null | HTMLDivElement>(null);
+  const tabs = useTabs();
+  const scrollTabs = useCallback(
+    (direction: number) => {
+      if (containerRef.current) {
+        const container = containerRef.current;
+        container.scrollLeft += direction * 100; // Adjust the scroll amount as needed
+        if (container.scrollLeft === 0) {
+          setHiddenGradient('left');
+        } else if (
+          container.scrollLeft + container.offsetWidth ===
+          container.scrollWidth
+        ) {
+          setHiddenGradient('right');
+        } else {
+          setHiddenGradient(undefined);
+        }
+      }
+    },
+    [containerRef.current],
+  );
+  return (
+    <div className="relative mt-6 flex flex-row items-center space-x-1 md:space-x-0 overflow-auto">
+      <div
+        className={`${hiddenGradient === 'left' ? 'hidden' : ''} md:hidden absolute inset-y-0 left-6 w-14 bg-gradient-to-r from-white dark:from-raisin-black pointer-events-none`}
+      ></div>
+      <div
+        className={`${hiddenGradient === 'right' ? 'hidden' : ''} md:hidden absolute inset-y-0 right-6 w-14 bg-gradient-to-l from-white dark:from-raisin-black pointer-events-none`}
+      ></div>
+
+      <div className="md:hidden z-10">
+        <button
+          onClick={() => scrollTabs(-1)}
+          className="flex h-6 w-6 items-center rounded-full justify-center transition-all hover:bg-gray-100"
+        >
+          <img src={ArrowLeft} alt="left-arrow" className="h-3" />
+        </button>
+      </div>
+      <div
+        ref={containerRef}
+        className="flex flex-nowrap overflow-x-auto no-scrollbar md:space-x-4 scroll-smooth snap-x"
+      >
+        {tabs.map((tab, index) => (
+          <button
+            key={index}
+            onClick={() => setActiveTab(tab)}
+            className={`snap-start h-9 rounded-3xl px-4 font-bold hover:text-neutral-600 dark:hover:text-white/60 ${
+              activeTab === tab
+                ? 'bg-neutral-100 text-neutral-600  dark:bg-dark-charcoal dark:text-white/60'
+                : 'text-gray-6000'
+            }`}
+          >
+            {tab}
+          </button>
+        ))}
+      </div>
+      <div className="md:hidden z-10">
+        <button
+          onClick={() => scrollTabs(1)}
+          className="flex h-6 w-6 rounded-full items-center justify-center hover:bg-gray-100"
+        >
+          <img src={ArrowRight} alt="right-arrow" className="h-3" />
+        </button>
+      </div>
+    </div>
+  );
+};
+
+export default SettingsBar;

frontend/src/components/ShareButton.tsx

@@ -0,0 +1,36 @@
+import { useState } from 'react';
+import ShareIcon from '../assets/share.svg';
+import { ShareConversationModal } from '../modals/ShareConversationModal';
+
+type ShareButtonProps = {
+  conversationId: string;
+};
+
+export default function ShareButton({ conversationId }: ShareButtonProps) {
+  const [isShareModalOpen, setShareModalState] = useState<boolean>(false);
+  return (
+    <>
+      <button
+        title="Share"
+        onClick={() => {
+          setShareModalState(true);
+        }}
+        className="absolute top-4 right-20 z-20 rounded-full hover:bg-bright-gray dark:hover:bg-[#28292E]"
+      >
+        <img
+          className="m-2 h-5 w-5 filter dark:invert"
+          alt="share"
+          src={ShareIcon}
+        />
+      </button>
+      {isShareModalOpen && (
+        <ShareConversationModal
+          close={() => {
+            setShareModalState(false);
+          }}
+          conversationId={conversationId}
+        />
+      )}
+    </>
+  );
+}

frontend/src/components/SkeletonLoader.tsx

@@ -0,0 +1,138 @@
+import React, { useState, useEffect } from 'react';
+
+interface SkeletonLoaderProps {
+  count?: number;
+  component?: 'default' | 'analysis' | 'chatbot' | 'logs';
+}
+
+const SkeletonLoader: React.FC<SkeletonLoaderProps> = ({
+  count = 1,
+  component = 'default',
+}) => {
+  const [skeletonCount, setSkeletonCount] = useState(count);
+
+  useEffect(() => {
+    const handleResize = () => {
+      const windowWidth = window.innerWidth;
+
+      if (windowWidth > 1024) {
+        setSkeletonCount(1);
+      } else if (windowWidth > 768) {
+        setSkeletonCount(count);
+      } else {
+        setSkeletonCount(Math.min(count, 2));
+      }
+    };
+
+    handleResize();
+    window.addEventListener('resize', handleResize);
+
+    return () => {
+      window.removeEventListener('resize', handleResize);
+    };
+  }, [count]);
+
+  return (
+    <div className="flex flex-col space-y-4">
+      {component === 'default' ? (
+        [...Array(skeletonCount)].map((_, idx) => (
+          <div
+            key={idx}
+            className={`p-6 ${skeletonCount === 1 ? 'w-full' : 'w-60'} dark:bg-raisin-black rounded-3xl animate-pulse`}
+          >
+            <div className="space-y-4">
+              <div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-3/4"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-5/6"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-1/2"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-3/4"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-full"></div>
+              </div>
+              <div className="border-t border-gray-600 my-4"></div>
+              <div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-2/3"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-1/4"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-full"></div>
+              </div>
+              <div className="border-t border-gray-600 my-4"></div>
+              <div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-5/6"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-1/3"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-2/3"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-full"></div>
+              </div>
+              <div className="border-t border-gray-600 my-4"></div>
+              <div className="h-4 bg-gray-600 rounded w-3/4 mb-2"></div>
+              <div className="h-4 bg-gray-600 rounded w-5/6 mb-2"></div>
+            </div>
+          </div>
+        ))
+      ) : component === 'analysis' ? (
+        [...Array(skeletonCount)].map((_, idx) => (
+          <div
+            key={idx}
+            className="p-6 w-full dark:bg-raisin-black rounded-3xl animate-pulse"
+          >
+            <div className="space-y-6">
+              <div className="space-y-2">
+                <div className="h-4 bg-gray-600 rounded w-1/3 mb-4"></div>
+                <div className="grid grid-cols-6 gap-2 items-end">
+                  <div className="h-32 bg-gray-600 rounded"></div>
+                  <div className="h-24 bg-gray-600 rounded"></div>
+                  <div className="h-40 bg-gray-600 rounded"></div>
+                  <div className="h-28 bg-gray-600 rounded"></div>
+                  <div className="h-36 bg-gray-600 rounded"></div>
+                  <div className="h-20 bg-gray-600 rounded"></div>
+                </div>
+              </div>
+              <div className="space-y-2">
+                <div className="h-4 bg-gray-600 rounded w-1/4 mb-4"></div>
+                <div className="h-32 bg-gray-600 rounded"></div>
+              </div>
+              <div className="grid grid-cols-2 gap-4">
+                <div className="h-4 bg-gray-600 rounded w-full"></div>
+                <div className="h-4 bg-gray-600 rounded w-full"></div>
+              </div>
+            </div>
+          </div>
+        ))
+      ) : component === 'chatbot' ? (
+        <div className="space-y-2 p-6 w-full dark:bg-raisin-black rounded-3xl animate-pulse">
+          <div className="grid grid-cols-4 gap-2 p-2">
+            <div className="h-4 bg-gray-600 rounded w-full"></div>
+            <div className="h-4 bg-gray-600 rounded w-full"></div>
+            <div className="h-4 bg-gray-600 rounded w-full"></div>
+            <div className="h-4 bg-gray-600 rounded w-full"></div>
+          </div>
+          <div className="border-t border-gray-600 my-2"></div>
+
+          {[...Array(skeletonCount * 6)].map((_, idx) => (
+            <div key={idx} className="grid grid-cols-4 gap-2 p-2 space-x-2">
+              <div className="h-4 bg-gray-500 rounded w-full"></div>
+              <div className="h-4 bg-gray-500 rounded w-full"></div>
+              <div className="h-4 bg-gray-500 rounded w-full"></div>
+              <div className="h-4 bg-gray-500 rounded w-full"></div>
+            </div>
+          ))}
+        </div>
+      ) : (
+        [...Array(skeletonCount)].map((_, idx) => (
+          <div
+            key={idx}
+            className="p-6 w-full dark:bg-raisin-black rounded-3xl animate-pulse"
+          >
+            <div className="space-y-4">
+              <div className="h-4 bg-gray-600 rounded w-1/2"></div>
+              <div className="h-4 bg-gray-600 rounded w-5/6"></div>
+              <div className="h-4 bg-gray-600 rounded w-3/4"></div>
+              <div className="h-4 bg-gray-600 rounded w-2/3"></div>
+              <div className="h-4 bg-gray-600 rounded w-1/4"></div>
+            </div>
+          </div>
+        ))
+      )}
+    </div>
+  );
+};
+
+export default SkeletonLoader;

frontend/src/components/SourceDropdown.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import Trash from '../assets/trash.svg';
 import Arrow2 from '../assets/dropdown-arrow.svg';
-import { Doc } from '../preferences/preferenceApi';
+import { Doc } from '../models/misc';
 import { useDispatch } from 'react-redux';
 import { useTranslation } from 'react-i18next';
 type Props = {
@@ -11,6 +11,7 @@ type Props = {
   isDocsListOpen: boolean;
   setIsDocsListOpen: React.Dispatch<React.SetStateAction<boolean>>;
   handleDeleteClick: any;
+  handlePostDocumentSelect: any;
 };
 
 function SourceDropdown({
@@ -20,6 +21,7 @@ function SourceDropdown({
   setIsDocsListOpen,
   isDocsListOpen,
   handleDeleteClick,
+  handlePostDocumentSelect, // Callback function fired after a document is selected
 }: Props) {
   const dispatch = useDispatch();
   const { t } = useTranslation();
@@ -63,9 +65,6 @@ function SourceDropdown({
             <p className="max-w-3/4 truncate whitespace-nowrap">
               {selectedDocs?.name || 'None'}
             </p>
-            <p className="flex flex-col items-center justify-center">
-              {selectedDocs?.version}
-            </p>
           </div>
         </span>
         <img
@@ -88,6 +87,7 @@ function SourceDropdown({
                     onClick={() => {
                       dispatch(setSelectedDocs(option));
                       setIsDocsListOpen(false);
+                      handlePostDocumentSelect(option);
                     }}
                   >
                     <span
@@ -121,7 +121,12 @@ function SourceDropdown({
             className="flex cursor-pointer items-center justify-between hover:bg-gray-100 dark:text-bright-gray dark:hover:bg-purple-taupe"
             onClick={handleEmptyDocumentSelect}
           >
-            <span className="ml-4 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3">
+            <span
+              className="ml-4 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3"
+              onClick={() => {
+                handlePostDocumentSelect(null);
+              }}
+            >
               {t('none')}
             </span>
           </div>

frontend/src/components/types/index.ts

@@ -2,6 +2,7 @@ export type InputProps = {
   type: 'text' | 'number';
   value: string | string[] | number;
   colorVariant?: 'silver' | 'jet' | 'gray';
+  borderVariant?: 'thin' | 'thick';
   isAutoFocused?: boolean;
   id?: string;
   maxLength?: number;

frontend/src/conversation/Conversation.tsx

@@ -5,13 +5,11 @@ import { useDispatch, useSelector } from 'react-redux';
 import ArrowDown from '../assets/arrow-down.svg';
 import Send from '../assets/send.svg';
 import SendDark from '../assets/send_dark.svg';
-import ShareIcon from '../assets/share.svg';
 import SpinnerDark from '../assets/spinner-dark.svg';
 import Spinner from '../assets/spinner.svg';
 import RetryIcon from '../components/RetryIcon';
 import Hero from '../Hero';
-import { useDarkTheme } from '../hooks';
-import { ShareConversationModal } from '../modals/ShareConversationModal';
+import { useDarkTheme, useMediaQuery } from '../hooks';
 import { selectConversationId } from '../preferences/preferenceSlice';
 import { AppDispatch } from '../store';
 import ConversationBubble from './ConversationBubble';
@@ -24,6 +22,7 @@ import {
   selectStatus,
   updateQuery,
 } from './conversationSlice';
+import ShareButton from '../components/ShareButton';
 
 export default function Conversation() {
   const queries = useSelector(selectQueries);
@@ -37,8 +36,8 @@ export default function Conversation() {
   const fetchStream = useRef<any>(null);
   const [eventInterrupt, setEventInterrupt] = useState(false);
   const [lastQueryReturnedErr, setLastQueryReturnedErr] = useState(false);
-  const [isShareModalOpen, setShareModalState] = useState<boolean>(false);
   const { t } = useTranslation();
+  const { isMobile } = useMediaQuery();
 
   const handleUserInterruption = () => {
     if (!eventInterrupt && status === 'loading') setEventInterrupt(true);
@@ -54,10 +53,6 @@ export default function Conversation() {
     }
   }, []);
 
-  useEffect(() => {
-    fetchStream.current && fetchStream.current.abort();
-  }, [conversationId]);
-
   useEffect(() => {
     if (queries.length) {
       queries[queries.length - 1].error && setLastQueryReturnedErr(true);
@@ -143,7 +138,7 @@ export default function Conversation() {
     } else if (query.error) {
       const retryBtn = (
         <button
-          className="flex items-center justify-center gap-3 self-center rounded-full border border-silver py-3 px-5  text-lg text-gray-500 transition-colors delay-100 hover:border-gray-500 disabled:cursor-not-allowed dark:text-bright-gray"
+          className="flex items-center justify-center gap-3 self-center rounded-full py-3 px-5  text-lg text-gray-500 transition-colors delay-100 hover:border-gray-500 disabled:cursor-not-allowed dark:text-bright-gray"
           disabled={status === 'loading'}
           onClick={() => {
             handleQuestion({
@@ -153,10 +148,12 @@ export default function Conversation() {
           }}
         >
           <RetryIcon
+            width={isMobile ? 12 : 12} // change the width and height according to device size if necessary
+            height={isMobile ? 12 : 12}
             fill={isDarkTheme ? 'rgb(236 236 241)' : 'rgb(107 114 120)'}
             stroke={isDarkTheme ? 'rgb(236 236 241)' : 'rgb(107 114 120)'}
+            strokeWidth={10}
           />
-          Retry
         </button>
       );
       responseView = (
@@ -182,46 +179,33 @@ export default function Conversation() {
       )}px`;
     }
   };
-
+  const checkScroll = () => {
+    const el = conversationRef.current;
+    if (!el) return;
+    const isBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 10;
+    setHasScrolledToLast(isBottom);
+  };
   useEffect(() => {
     handleInput();
     window.addEventListener('resize', handleInput);
+    conversationRef.current?.addEventListener('scroll', checkScroll);
     return () => {
       window.removeEventListener('resize', handleInput);
+      conversationRef.current?.removeEventListener('scroll', checkScroll);
     };
   }, []);
   return (
-    <div className="flex h-[90vh] flex-col gap-7 pb-2 sm:h-[85vh]">
+    <div className="flex flex-col gap-1 h-full justify-end">
       {conversationId && (
-        <>
-          <button
-            title="Share"
-            onClick={() => {
-              setShareModalState(true);
-            }}
-            className="fixed top-4 right-20 z-0 rounded-full hover:bg-bright-gray dark:hover:bg-[#28292E]"
-          >
-            <img
-              className="m-2 h-5 w-5 filter dark:invert"
-              alt="share"
-              src={ShareIcon}
-            />
-          </button>
-          {isShareModalOpen && (
-            <ShareConversationModal
-              close={() => {
-                setShareModalState(false);
-              }}
-              conversationId={conversationId}
-            />
-          )}
-        </>
+        <div className="hidden md:block">
+          <ShareButton conversationId={conversationId} />
+        </div>
       )}
       <div
         ref={conversationRef}
         onWheel={handleUserInterruption}
         onTouchMove={handleUserInterruption}
-        className="flex h-[90%] w-full flex-1 justify-center overflow-y-auto p-4 md:h-[83vh]"
+        className="flex justify-center w-full overflow-y-auto h-screen sm:mt-12"
       >
         {queries.length > 0 && !hasScrolledToLast && (
           <button
@@ -237,13 +221,13 @@ export default function Conversation() {
           </button>
         )}
 
-        {queries.length > 0 && (
-          <div className="mt-16 w-full md:w-8/12">
+        {queries.length > 0 ? (
+          <div className="w-full md:w-8/12">
             {queries.map((query, index) => {
               return (
                 <Fragment key={index}>
                   <ConversationBubble
-                    className={'mb-1 last:mb-28 md:mb-7'}
+                    className={'first:mt-5'}
                     key={`${index}QUESTION`}
                     message={query.prompt}
                     type="QUESTION"
@@ -255,12 +239,12 @@ export default function Conversation() {
               );
             })}
           </div>
+        ) : (
+          <Hero handleQuestion={handleQuestion} />
         )}
-
-        {queries.length === 0 && <Hero handleQuestion={handleQuestion} />}
       </div>
 
-      <div className="bottom-safe fixed flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 pb-1 sm:w-1/2">
+      <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 pb-1 sm:w-[62%] h-auto">
         <div className="flex w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black">
           <textarea
             id="inputbox"

frontend/src/conversation/ConversationBubble.tsx

@@ -4,8 +4,10 @@ import { useSelector } from 'react-redux';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { vscDarkPlus } from 'react-syntax-highlighter/dist/cjs/styles/prism';
 import remarkGfm from 'remark-gfm';
+import remarkMath from 'remark-math';
+import rehypeKatex from 'rehype-katex';
+import 'katex/dist/katex.min.css';
 
-import Alert from '../assets/alert.svg';
 import DocsGPT3 from '../assets/cute_docsgpt3.svg';
 import Dislike from '../assets/dislike.svg?react';
 import Document from '../assets/document.svg';
@@ -51,16 +53,37 @@ const ConversationBubble = forwardRef<
   let bubble;
   if (type === 'QUESTION') {
     bubble = (
-      <div ref={ref} className={`flex flex-row-reverse self-end ${className}`}>
+      <div
+        ref={ref}
+        className={`flex flex-row-reverse self-end flex-wrap ${className}`}
+      >
         <Avatar className="mt-2 text-2xl" avatar="🧑‍💻"></Avatar>
-        <div className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white">
-          <ReactMarkdown className="whitespace-pre-wrap break-normal leading-normal">
-            {message}
-          </ReactMarkdown>
+        <div
+          style={{
+            wordBreak: 'break-word',
+          }}
+          className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white max-w-full whitespace-pre-wrap leading-normal"
+        >
+          {message}
         </div>
       </div>
     );
   } else {
+    const preprocessLaTeX = (content: string) => {
+      // Replace block-level LaTeX delimiters \[ \] with $$ $$
+      const blockProcessedContent = content.replace(
+        /\\\[(.*?)\\\]/gs,
+        (_, equation) => `$$${equation}$$`,
+      );
+
+      // Replace inline LaTeX delimiters \( \) with $ $
+      const inlineProcessedContent = blockProcessedContent.replace(
+        /\\\((.*?)\\\)/gs,
+        (_, equation) => `$${equation}$`,
+      );
+
+      return inlineProcessedContent;
+    };
     bubble = (
       <div
         ref={ref}
@@ -180,7 +203,7 @@ const ConversationBubble = forwardRef<
                   ))}
                   {(sources?.length ?? 0) > 3 && (
                     <div
-                      className="flex h-24 cursor-pointer flex-col-reverse rounded-[20px] bg-gray-1000 p-4 text-purple-30 hover:bg-[#F1F1F1] hover:text-[#6D3ECC] dark:bg-gun-metal dark:hover:bg-[#2C2E3C] dark:hover:text-[#8C67D7]"
+                      className="flex h-28 cursor-pointer flex-col-reverse rounded-[20px] bg-gray-1000 p-4 text-purple-30 hover:bg-[#F1F1F1] hover:text-[#6D3ECC] dark:bg-gun-metal dark:hover:bg-[#2C2E3C] dark:hover:text-[#8C67D7]"
                       onClick={() => setIsSidebarOpen(true)}
                     >
                       <p className="ellipsis-text h-22 text-xs">{`View ${
@@ -214,17 +237,10 @@ const ConversationBubble = forwardRef<
                 : 'flex-col rounded-3xl'
             }`}
           >
-            {type === 'ERROR' && (
-              <>
-                <img src={Alert} alt="alert" className="mr-2 inline" />
-                <div className="absolute -right-32 top-1/2 -translate-y-1/2">
-                  {retryBtn}
-                </div>
-              </>
-            )}
             <ReactMarkdown
               className="whitespace-pre-wrap break-normal leading-normal"
-              remarkPlugins={[remarkGfm]}
+              remarkPlugins={[remarkGfm, remarkMath]}
+              rehypePlugins={[rehypeKatex]}
               components={{
                 code(props) {
                   const { children, className, node, ref, ...rest } = props;
@@ -307,19 +323,24 @@ const ConversationBubble = forwardRef<
                 },
               }}
             >
-              {message}
+              {preprocessLaTeX(message)}
             </ReactMarkdown>
           </div>
         </div>
         <div className="my-2 ml-2 flex justify-start">
           <div
             className={`relative mr-5  block items-center justify-center lg:invisible 
-            ${type !== 'ERROR' ? 'group-hover:lg:visible' : ''}`}
+            ${type !== 'ERROR' ? 'group-hover:lg:visible' : 'hidden'}`}
           >
             <div>
               <CopyButton text={message} />
             </div>
           </div>
+          {type === 'ERROR' && (
+            <div className="relative mr-5 block items-center justify-center">
+              <div>{retryBtn}</div>
+            </div>
+          )}
           {handleFeedback && (
             <>
               <div

frontend/src/conversation/ConversationTile.tsx

@@ -1,9 +1,13 @@
-import { SyntheticEvent, useEffect, useRef, useState } from 'react';
+import {
+  SyntheticEvent,
+  useEffect,
+  useRef,
+  useState,
+  useCallback,
+} from 'react';
 import { useSelector } from 'react-redux';
 import Edit from '../assets/edit.svg';
 import Exit from '../assets/exit.svg';
-import Message from '../assets/message.svg';
-import MessageDark from '../assets/message-dark.svg';
 import { useDarkTheme } from '../hooks';
 import ConfirmationModal from '../modals/ConfirmationModal';
 import CheckMark2 from '../assets/checkMark2.svg';
@@ -22,6 +26,7 @@ interface ConversationProps {
 interface ConversationTileProps {
   conversation: ConversationProps;
   selectConversation: (arg1: string) => void;
+  onCoversationClick: () => void; //Callback to handle click on conversation tile regardless of selected or not
   onDeleteConversation: (arg1: string) => void;
   onSave: ({ name, id }: ConversationProps) => void;
 }
@@ -29,6 +34,7 @@ interface ConversationTileProps {
 export default function ConversationTile({
   conversation,
   selectConversation,
+  onCoversationClick,
   onDeleteConversation,
   onSave,
 }: ConversationTileProps) {
@@ -75,6 +81,36 @@ export default function ConversationTile({
       document.removeEventListener('mousedown', handleClickOutside);
     };
   }, []);
+
+  const preventScroll = useCallback((event: WheelEvent | TouchEvent) => {
+    event.preventDefault();
+  }, []);
+
+  useEffect(() => {
+    const conversationsMainDiv = document.getElementById(
+      'conversationsMainDiv',
+    );
+
+    if (conversationsMainDiv) {
+      if (isOpen) {
+        conversationsMainDiv.addEventListener('wheel', preventScroll, {
+          passive: false,
+        });
+        conversationsMainDiv.addEventListener('touchmove', preventScroll, {
+          passive: false,
+        });
+      } else {
+        conversationsMainDiv.removeEventListener('wheel', preventScroll);
+        conversationsMainDiv.removeEventListener('touchmove', preventScroll);
+      }
+
+      return () => {
+        conversationsMainDiv.removeEventListener('wheel', preventScroll);
+        conversationsMainDiv.removeEventListener('touchmove', preventScroll);
+      };
+    }
+  }, [isOpen]);
+
   function onClear() {
     setConversationsName(conversation.name);
     setIsEdit(false);
@@ -90,20 +126,17 @@ export default function ConversationTile({
           setIsHovered(false);
         }}
         onClick={() => {
+          onCoversationClick();
           conversationId !== conversation.id &&
             selectConversation(conversation.id);
         }}
-        className={`my-auto mx-4 mt-4 flex h-9 cursor-pointer items-center justify-between gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
+        className={`my-auto mx-4 mt-4 flex h-9 cursor-pointer items-center justify-between pl-4 gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
           conversationId === conversation.id || isOpen || isHovered
             ? 'bg-gray-100 dark:bg-[#28292E]'
             : ''
         }`}
       >
         <div className={`flex w-10/12 gap-4`}>
-          <img
-            src={isDarkTheme ? MessageDark : Message}
-            className="ml-4 w-5 dark:text-white"
-          />
           {isEdit ? (
             <input
               autoFocus
@@ -150,7 +183,7 @@ export default function ConversationTile({
               <button
                 onClick={(event: SyntheticEvent) => {
                   event.stopPropagation();
-                  setOpen(true);
+                  setOpen(!isOpen);
                 }}
                 className="mr-2 flex w-4 justify-center"
               >
@@ -158,7 +191,12 @@ export default function ConversationTile({
               </button>
             )}
             {isOpen && (
-              <div className="flex-start absolute z-30 flex w-32 translate-x-1 translate-y-5 flex-col rounded-xl bg-stone-100 text-sm text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver md:w-36">
+              <div
+                className="flex-start absolute z-30 flex w-32 translate-x-1 translate-y-5 flex-col rounded-xl bg-stone-100 text-sm text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver md:w-36"
+                style={{
+                  top: `${(tileRef.current?.getBoundingClientRect().top ?? 0) + window.scrollY + 8}px`,
+                }}
+              >
                 <button
                   onClick={(event: SyntheticEvent) => {
                     event.stopPropagation();

frontend/src/conversation/SharedConversation.tsx

@@ -26,7 +26,6 @@ import {
   selectQueries,
 } from './sharedConversationSlice';
 import { useSelector } from 'react-redux';
-const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
 
 export const SharedConversation = () => {
   const navigate = useNavigate();
@@ -39,6 +38,7 @@ export const SharedConversation = () => {
   const status = useSelector(selectStatus);
 
   const inputRef = useRef<HTMLDivElement>(null);
+  const sharedConversationRef = useRef<HTMLDivElement>(null);
   const { t } = useTranslation();
   const dispatch = useDispatch<AppDispatch>();
 
@@ -60,38 +60,6 @@ export const SharedConversation = () => {
     }
   }, []);
 
-  function formatISODate(isoDateStr: string) {
-    const date = new Date(isoDateStr);
-
-    const monthNames = [
-      'Jan',
-      'Feb',
-      'Mar',
-      'Apr',
-      'May',
-      'June',
-      'July',
-      'Aug',
-      'Sept',
-      'Oct',
-      'Nov',
-      'Dec',
-    ];
-
-    const month = monthNames[date.getMonth()];
-    const day = date.getDate();
-    const year = date.getFullYear();
-
-    let hours = date.getHours();
-    const minutes = date.getMinutes();
-    const ampm = hours >= 12 ? 'PM' : 'AM';
-
-    hours = hours % 12;
-    hours = hours ? hours : 12;
-    const minutesStr = minutes < 10 ? '0' + minutes : minutes;
-    const formattedDate = `Published ${month} ${day}, ${year} at ${hours}:${minutesStr} ${ampm}`;
-    return formattedDate;
-  }
   useEffect(() => {
     if (queries.length) {
       queries[queries.length - 1].error && setLastQueryReturnedErr(true);
@@ -100,10 +68,17 @@ export const SharedConversation = () => {
   }, [queries[queries.length - 1]]);
 
   const scrollIntoView = () => {
-    endMessageRef?.current?.scrollIntoView({
-      behavior: 'smooth',
-      block: 'start',
-    });
+    if (!sharedConversationRef?.current || eventInterrupt) return;
+
+    if (status === 'idle' || !queries[queries.length - 1].response) {
+      sharedConversationRef.current.scrollTo({
+        behavior: 'smooth',
+        top: sharedConversationRef.current.scrollHeight,
+      });
+    } else {
+      sharedConversationRef.current.scrollTop =
+        sharedConversationRef.current.scrollHeight;
+    }
   };
 
   const fetchQueries = () => {
@@ -144,6 +119,7 @@ export const SharedConversation = () => {
           key={`${index}ANSWER`}
           message={query.response}
           type={'ANSWER'}
+          sources={query.sources ?? []}
         ></ConversationBubble>
       );
     } else if (query.error) {
@@ -201,7 +177,12 @@ export const SharedConversation = () => {
 
   return (
     <div className="flex h-full flex-col items-center justify-between gap-2 overflow-y-hidden dark:bg-raisin-black">
-      <div className="flex w-full justify-center overflow-auto">
+      <div
+        ref={sharedConversationRef}
+        onWheel={handleUserInterruption}
+        onTouchMove={handleUserInterruption}
+        className="flex w-full justify-center overflow-auto"
+      >
         <div className="mt-0 w-11/12 md:w-10/12 lg:w-6/12">
           <div className="mb-2 w-full border-b pb-2 dark:border-b-silver">
             <h1 className="font-semi-bold text-4xl text-chinese-black dark:text-chinese-silver">

frontend/src/conversation/conversationHandlers.ts

@@ -1,32 +1,6 @@
 import conversationService from '../api/services/conversationService';
-import { Doc } from '../preferences/preferenceApi';
-import { Answer, FEEDBACK } from './conversationModels';
-
-function getDocPath(selectedDocs: Doc | null): string {
-  let docPath = 'default';
-  if (selectedDocs) {
-    let namePath = selectedDocs.name;
-    if (selectedDocs.language === namePath) {
-      namePath = '.project';
-    }
-    if (selectedDocs.location === 'local') {
-      docPath = 'local' + '/' + selectedDocs.name + '/';
-    } else if (selectedDocs.location === 'remote') {
-      docPath =
-        selectedDocs.language +
-        '/' +
-        namePath +
-        '/' +
-        selectedDocs.version +
-        '/' +
-        selectedDocs.model +
-        '/';
-    } else if (selectedDocs.location === 'custom') {
-      docPath = selectedDocs.docLink;
-    }
-  }
-  return docPath;
-}
+import { Doc } from '../models/misc';
+import { Answer, FEEDBACK, RetrievalPayload } from './conversationModels';
 
 export function handleFetchAnswer(
   question: string,
@@ -54,23 +28,24 @@ export function handleFetchAnswer(
       title: any;
     }
 > {
-  const docPath = getDocPath(selectedDocs);
   history = history.map((item) => {
     return { prompt: item.prompt, response: item.response };
   });
+  const payload: RetrievalPayload = {
+    question: question,
+    history: JSON.stringify(history),
+    conversation_id: conversationId,
+    prompt_id: promptId,
+    chunks: chunks,
+    token_limit: token_limit,
+    isNoneDoc: selectedDocs === null,
+  };
+  if (selectedDocs && 'id' in selectedDocs) {
+    payload.active_docs = selectedDocs.id as string;
+  }
+  payload.retriever = selectedDocs?.retriever as string;
   return conversationService
-    .answer(
-      {
-        question: question,
-        history: history,
-        active_docs: docPath,
-        conversation_id: conversationId,
-        prompt_id: promptId,
-        chunks: chunks,
-        token_limit: token_limit,
-      },
-      signal,
-    )
+    .answer(payload, signal)
     .then((response) => {
       if (response.ok) {
         return response.json();
@@ -101,25 +76,26 @@ export function handleFetchAnswerSteaming(
   token_limit: number,
   onEvent: (event: MessageEvent) => void,
 ): Promise<Answer> {
-  const docPath = getDocPath(selectedDocs);
   history = history.map((item) => {
     return { prompt: item.prompt, response: item.response };
   });
+  const payload: RetrievalPayload = {
+    question: question,
+    history: JSON.stringify(history),
+    conversation_id: conversationId,
+    prompt_id: promptId,
+    chunks: chunks,
+    token_limit: token_limit,
+    isNoneDoc: selectedDocs === null,
+  };
+  if (selectedDocs && 'id' in selectedDocs) {
+    payload.active_docs = selectedDocs.id as string;
+  }
+  payload.retriever = selectedDocs?.retriever as string;
+
   return new Promise<Answer>((resolve, reject) => {
     conversationService
-      .answerStream(
-        {
-          question: question,
-          active_docs: docPath,
-          history: JSON.stringify(history),
-          conversation_id: conversationId,
-          prompt_id: promptId,
-          chunks: chunks,
-          token_limit: token_limit,
-          isNoneDoc: selectedDocs === null,
-        },
-        signal,
-      )
+      .answerStream(payload, signal)
       .then((response) => {
         if (!response.body) throw Error('No response body');
 
@@ -176,16 +152,42 @@ export function handleSearch(
   chunks: string,
   token_limit: number,
 ) {
-  const docPath = getDocPath(selectedDocs);
+  history = history.map((item) => {
+    return { prompt: item.prompt, response: item.response };
+  });
+  const payload: RetrievalPayload = {
+    question: question,
+    history: JSON.stringify(history),
+    conversation_id: conversation_id,
+    chunks: chunks,
+    token_limit: token_limit,
+    isNoneDoc: selectedDocs === null,
+  };
+  if (selectedDocs && 'id' in selectedDocs)
+    payload.active_docs = selectedDocs.id as string;
+  payload.retriever = selectedDocs?.retriever as string;
+  return conversationService
+    .search(payload)
+    .then((response) => response.json())
+    .then((data) => {
+      return data;
+    })
+    .catch((err) => console.log(err));
+}
+
+export function handleSearchViaApiKey(
+  question: string,
+  api_key: string,
+  history: Array<any> = [],
+) {
+  history = history.map((item) => {
+    return { prompt: item.prompt, response: item.response };
+  });
   return conversationService
     .search({
       question: question,
-      active_docs: docPath,
-      conversation_id,
-      history,
-      chunks: chunks,
-      token_limit: token_limit,
-      isNoneDoc: selectedDocs === null,
+      history: JSON.stringify(history),
+      api_key: api_key,
     })
     .then((response) => response.json())
     .then((data) => {

frontend/src/conversation/conversationModels.ts

@@ -31,3 +31,14 @@ export interface Query {
   conversationId?: string | null;
   title?: string | null;
 }
+export interface RetrievalPayload {
+  question: string;
+  active_docs?: string;
+  retriever?: string;
+  history: string;
+  conversation_id: string | null;
+  prompt_id?: string | null;
+  chunks: string;
+  token_limit: number;
+  isNoneDoc: boolean;
+}

frontend/src/conversation/conversationSlice.ts

@@ -6,7 +6,6 @@ import store from '../store';
 import {
   handleFetchAnswer,
   handleFetchAnswerSteaming,
-  handleSearch,
 } from './conversationHandlers';
 import { Answer, ConversationState, Query, Status } from './conversationModels';
 
@@ -48,30 +47,19 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
                 .catch((error) => {
                   console.error('Failed to fetch conversations: ', error);
                 });
-
-              handleSearch(
-                //search for sources post streaming
-                question,
-                state.preference.selectedDocs!,
-                state.conversation.conversationId,
-                state.conversation.queries,
-                state.preference.chunks,
-                state.preference.token_limit,
-              ).then((sources) => {
-                //dispatch streaming sources
-                dispatch(
-                  updateStreamingSource({
-                    index: state.conversation.queries.length - 1,
-                    query: { sources: sources ?? [] },
-                  }),
-                );
-              });
             } else if (data.type === 'id') {
               dispatch(
                 updateConversationId({
                   query: { conversationId: data.id },
                 }),
               );
+            } else if (data.type === 'source') {
+              dispatch(
+                updateStreamingSource({
+                  index: state.conversation.queries.length - 1,
+                  query: { sources: data.source ?? [] },
+                }),
+              );
             } else if (data.type === 'error') {
               // set status to 'failed'
               dispatch(conversationSlice.actions.setStatus('failed'));
@@ -163,6 +151,7 @@ export const conversationSlice = createSlice({
       state,
       action: PayloadAction<{ index: number; query: Partial<Query> }>,
     ) {
+      if (state.status === 'idle') return;
       const { index, query } = action.payload;
       if (query.response != undefined) {
         state.queries[index].response =
@@ -179,6 +168,7 @@ export const conversationSlice = createSlice({
       action: PayloadAction<{ query: Partial<Query> }>,
     ) {
       state.conversationId = action.payload.query.conversationId ?? null;
+      state.status = 'idle';
     },
     updateStreamingSource(
       state,

frontend/src/conversation/sharedConversationSlice.ts

@@ -44,6 +44,13 @@ export const fetchSharedAnswer = createAsyncThunk<Answer, { question: string }>(
               // set status to 'idle'
               dispatch(sharedConversationSlice.actions.setStatus('idle'));
               dispatch(saveToLocalStorage());
+            } else if (data.type === 'source') {
+              dispatch(
+                updateStreamingSource({
+                  index: state.sharedConversation.queries.length - 1,
+                  query: { sources: data.source ?? [] },
+                }),
+              );
             } else if (data.type === 'error') {
               // set status to 'failed'
               dispatch(sharedConversationSlice.actions.setStatus('failed'));
@@ -164,6 +171,20 @@ export const sharedConversationSlice = createSlice({
         ...query,
       };
     },
+    updateStreamingSource(
+      state,
+      action: PayloadAction<{ index: number; query: Partial<Query> }>,
+    ) {
+      const { index, query } = action.payload;
+      if (!state.queries[index].sources) {
+        state.queries[index].sources = query.sources ?? [];
+      } else if (query.sources && query.sources.length > 0) {
+        state.queries[index].sources = [
+          ...(state.queries[index].sources ?? []),
+          ...query.sources,
+        ];
+      }
+    },
     raiseError(
       state,
       action: PayloadAction<{ index: number; message: string }>,
@@ -213,9 +234,11 @@ export const {
   updateStreamingQuery,
   addQuery,
   saveToLocalStorage,
+  updateStreamingSource,
 } = sharedConversationSlice.actions;
 
-export const selectStatus = (state: RootState) => state.conversation.status;
+export const selectStatus = (state: RootState) =>
+  state.sharedConversation.status;
 export const selectClientAPIKey = (state: RootState) =>
   state.sharedConversation.apiKey;
 export const selectQueries = (state: RootState) =>

frontend/src/hooks/index.ts

@@ -66,9 +66,35 @@ export function useMediaQuery() {
 }
 
 export function useDarkTheme() {
-  const [isDarkTheme, setIsDarkTheme] = useState<boolean>(
-    localStorage.getItem('selectedTheme') === 'Dark' || false,
-  );
+  const getSystemThemePreference = () => {
+    return (
+      window.matchMedia &&
+      window.matchMedia('(prefers-color-scheme: dark)').matches
+    );
+  };
+
+  const getInitialTheme = () => {
+    const storedTheme = localStorage.getItem('selectedTheme');
+    if (storedTheme === 'Dark' || storedTheme === 'Light') {
+      return storedTheme === 'Dark';
+    }
+    return getSystemThemePreference();
+  };
+
+  const [isDarkTheme, setIsDarkTheme] = useState<boolean>(getInitialTheme());
+  const [componentMounted, setComponentMounted] = useState(false);
+
+  useEffect(() => {
+    const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)');
+    const handleChange = () => {
+      if (localStorage.getItem('selectedTheme') === null) {
+        setIsDarkTheme(mediaQuery.matches);
+      }
+    };
+
+    mediaQuery.addListener(handleChange);
+    return () => mediaQuery.removeListener(handleChange);
+  }, []);
 
   useEffect(() => {
     localStorage.setItem('selectedTheme', isDarkTheme ? 'Dark' : 'Light');
@@ -77,11 +103,12 @@ export function useDarkTheme() {
     } else {
       document.body?.classList.remove('dark');
     }
+    setComponentMounted(true);
   }, [isDarkTheme]);
 
-  const toggleTheme: any = () => {
+  const toggleTheme = () => {
     setIsDarkTheme(!isDarkTheme);
   };
 
-  return [isDarkTheme, toggleTheme];
+  return [isDarkTheme, toggleTheme, componentMounted] as const;
 }

frontend/src/hooks/useDefaultDocument.ts

@@ -1,7 +1,8 @@
 import React from 'react';
 import { useDispatch, useSelector } from 'react-redux';
 
-import { Doc, getDocs } from '../preferences/preferenceApi';
+import { getDocs } from '../preferences/preferenceApi';
+import { Doc } from '../models/misc';
 import {
   selectSelectedDocs,
   setSelectedDocs,

frontend/src/index.css

@@ -2,6 +2,16 @@
 @tailwind components;
 @tailwind utilities;
 
+:root {
+  --viewport-height: 100vh;
+}
+
+@supports (height: 100dvh) {
+  :root {
+    --viewport-height: 100dvh; /* Use dvh where supported */
+  }
+}
+
 body.dark {
   background-color: #202124; /* raisin-black */
 }
@@ -37,6 +47,28 @@ body.dark {
   }
 }
 
+@layer components {
+  .table-default {
+    @apply block w-max table-auto content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
+  }
+
+  .table-default th {
+    @apply border-r border-silver dark:border-silver/40 p-4 w-[244px];
+  }
+
+  .table-default th:last-child {
+    @apply w-[auto] border-r-0;
+  }
+
+  .table-default td {
+    @apply border-r border-t border-silver dark:border-silver/40 px-4 py-2;
+  }
+
+  .table-default td:last-child {
+    @apply border-r-0;
+  }
+}
+
 /*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */
 
 /* Document
@@ -63,11 +95,21 @@ html {
 
 body {
   margin: 0;
-  min-height: 100vh;
+  min-height: var(--viewport-height);
   overflow-x: hidden;
   font-family: 'Inter', sans-serif;
 }
-
+/*
+Avoid over-scrolling in mobile browsers
+*/
+@media only screen and (max-width: 500px) {
+  body,
+  html {
+    min-height: var(--viewport-height);
+    position: fixed;
+    width: 100%;
+  }
+}
 /**
  * Render the `main` element consistently in IE.
  */
@@ -420,8 +462,34 @@ template {
   src: url('/fonts/Inter-Variable.ttf');
 }
 
+@font-face {
+  font-family: 'IBMPlexMono-Medium';
+  font-weight: 500;
+  src: url('/fonts/IBMPlexMono-Medium.ttf');
+}
+
 ::-webkit-scrollbar {
-  width: 0;
+  width: 10;
+}
+
+input:-webkit-autofill {
+  -webkit-box-shadow: 0 0 0 50px white inset;
+}
+
+input:-webkit-autofill:focus {
+  -webkit-box-shadow: 0 0 0 50px white inset;
+}
+
+@media (prefers-color-scheme: dark) {
+  input:-webkit-autofill {
+    -webkit-box-shadow: 0 0 0 50px rgb(68, 70, 84) inset;
+    -webkit-text-fill-color: white;
+  }
+
+  input:-webkit-autofill:focus {
+    -webkit-box-shadow: 0 0 0 50px rgb(68, 70, 84) inset;
+    -webkit-text-fill-color: white;
+  }
 }
 
 .inputbox-style {
@@ -441,3 +509,7 @@ template {
   -webkit-box-orient: vertical;
   text-overflow: ellipsis;
 }
+
+.logs-table {
+  font-family: 'IBMPlexMono-Medium', system-ui;
+}

frontend/src/locale/en.json

@@ -7,9 +7,12 @@
   "about": "About",
   "inputPlaceholder": "Type your message here...",
   "tagline": "DocsGPT uses GenAI, please review critical information using sources.",
-  "sourceDocs": "Source Docs",
+  "sourceDocs": "Source",
   "none": "None",
   "cancel": "Cancel",
+  "help": "Help",
+  "emailUs": "Email us",
+  "documentation": "documentation",
   "demo": [
     {
       "header": "Learn about DocsGPT",
@@ -54,29 +57,42 @@
       "name": "Document Name",
       "date": "Vector Date",
       "type": "Type",
-      "tokenUsage": "Token Usage"
+      "tokenUsage": "Token Usage",
+      "noData": "No existing Documents"
     },
     "apiKeys": {
       "label": "Chatbots",
       "name": "Name",
       "key": "API Key",
       "sourceDoc": "Source Document",
-      "createNew": "Create New"
+      "createNew": "Create New",
+      "noData": "No existing Chatbots"
+    },
+    "analytics": {
+      "label": "Analytics"
+    },
+    "logs": {
+      "label": "Logs"
     }
   },
   "modals": {
     "uploadDoc": {
       "label": "Upload New Documentation",
-      "file": "From File",
-      "remote": "Remote",
+      "select": "Choose how to upload your document to DocsGPT",
+      "file": "Upload from device",
+      "back": "Back",
+      "wait": "Please wait ...",
+      "remote": "Collect from a website",
+      "start": "Start Chatting",
       "name": "Name",
       "choose": "Choose Files",
-      "info": "Please upload .pdf, .txt, .rst, .csv, .docx, .md, .zip limited to 25mb",
+      "info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .zip limited to 25mb",
       "uploadedFiles": "Uploaded Files",
       "cancel": "Cancel",
       "train": "Train",
       "link": "Link",
       "urlLink": "URL Link",
+      "repoUrl": "Repository URL",
       "reddit": {
         "id": "Client ID",
         "secret": "Client Secret",

frontend/src/locale/es.json

@@ -7,9 +7,12 @@
   "about": "Acerca de",
   "inputPlaceholder": "Escribe tu mensaje aquí...",
   "tagline": "DocsGPT utiliza GenAI, por favor revisa información crítica utilizando fuentes.",
-  "sourceDocs": "Documentos Fuente",
+  "sourceDocs": "Fuente",
   "none": "Nada",
   "cancel": "Cancelar",
+  "help": "Asistencia",
+  "emailUs": "Envíanos un correo",
+  "documentation": "documentación",
   "demo": [
     {
       "header": "Aprende sobre DocsGPT",
@@ -54,21 +57,33 @@
       "name": "Nombre del Documento",
       "date": "Fecha Vector",
       "type": "Tipo",
-      "tokenUsage": "Uso de Tokens"
+      "tokenUsage": "Uso de Tokens",
+      "noData": "No hay documentos existentes"
     },
     "apiKeys": {
       "label": "Chatbots",
       "name": "Nombre",
       "key": "Clave de API",
       "sourceDoc": "Documento Fuente",
-      "createNew": "Crear Nuevo"
+      "createNew": "Crear Nuevo",
+      "noData": "No hay chatbots existentes"
+    },
+    "analytics": {
+      "label": "Analítica"
+    },
+    "logs": {
+      "label": "Registros"
     }
   },
   "modals": {
     "uploadDoc": {
       "label": "Subir Nueva Documentación",
-      "file": "Desde Archivo",
-      "remote": "Remota",
+      "select": "Elija cómo cargar su documento en DocsGPT",
+      "file": "Subir desde el dispositivo",
+      "back": "Atrás",
+      "wait": "Espere por favor ...",
+      "remote": "Recoger desde un sitio web",
+      "start": "Empezar a chatear",
       "name": "Nombre",
       "choose": "Seleccionar Archivos",
       "info": "Por favor, suba archivos .pdf, .txt, .rst, .docx, .md, .zip limitados a 25 MB",
@@ -77,6 +92,7 @@
       "train": "Entrenar",
       "link": "Enlace",
       "urlLink": "Enlace URL",
+      "repoUrl": "URL del Repositorio",
       "reddit": {
         "id": "ID de Cliente",
         "secret": "Secreto de Cliente",

frontend/src/locale/i18n.ts

@@ -6,6 +6,7 @@ import en from './en.json'; //English
 import es from './es.json'; //Spanish
 import jp from './jp.json'; //Japanese
 import zh from './zh.json'; //Mandarin
+import zhTW from './zh-TW.json'; //Traditional Chinese
 
 i18n
   .use(LanguageDetector)
@@ -24,6 +25,9 @@ i18n
       zh: {
         translation: zh,
       },
+      'zh-TW': {
+        translation: zhTW,
+      },
     },
     fallbackLng: 'en',
     detection: {

frontend/src/locale/jp.json

@@ -7,9 +7,12 @@
   "about": "について",
   "inputPlaceholder": "ここにメッセージを入力してください...",
   "tagline": "DocsGPTはGenAIを使用しています。重要な情報はソースで確認してください。",
-  "sourceDocs": "ソースドキュメント",
+  "sourceDocs": "ソース",
   "none": "なし",
   "cancel": "キャンセル",
+  "help": "ヘルプ",
+  "emailUs": "メールを送る",
+  "documentation": "ドキュメント",
   "demo": [
     {
       "header": "DocsGPTについて学ぶ",
@@ -54,21 +57,33 @@
       "name": "ドキュメント名",
       "date": "ベクトル日付",
       "type": "タイプ",
-      "tokenUsage": "トークン使用量"
+      "tokenUsage": "トークン使用量",
+      "noData": "既存のドキュメントはありません"
     },
     "apiKeys": {
       "label": "チャットボット",
       "name": "名前",
       "key": "APIキー",
       "sourceDoc": "ソースドキュメント",
-      "createNew": "新規作成"
+      "createNew": "新規作成",
+      "noData": "既存のチャットボットはありません"
+    },
+    "analytics": {
+      "label": "分析"
+    },
+    "logs": {
+      "label": "ログ"
     }
   },
   "modals": {
     "uploadDoc": {
       "label": "新規書類のアップロード",
-      "file": "ファイルから",
-      "remote": "リモート",
+      "select": "ドキュメントを DocsGPT にアップロードする方法を選択します",
+      "file": "デバイスからアップロード",
+      "back": "戻る",
+      "wait": "お待ちください ...",
+      "remote": "ウェブサイトから収集する",
+      "start": "チャットを開始する",
       "name": "名前",
       "choose": "ファイルを選択",
       "info": ".pdf, .txt, .rst, .docx, .md, .zipファイルを25MBまでアップロードしてください",
@@ -77,6 +92,7 @@
       "train": "トレーニング",
       "link": "リンク",
       "urlLink": "URLリンク",
+      "repoUrl": "リポジトリURL",
       "reddit": {
         "id": "クライアントID",
         "secret": "クライアントシークレット",

frontend/src/locale/zh-TW.json

@@ -0,0 +1,133 @@
+{
+  "language": "繁體中文（臺灣）",
+  "chat": "對話",
+  "chats": "對話",
+  "newChat": "新對話",
+  "myPlan": "我的方案",
+  "about": "關於",
+  "inputPlaceholder": "在此輸入您的訊息...",
+  "tagline": "DocsGPT 使用生成式 AI，請使用原始資料來源審閱重要資訊。",
+  "sourceDocs": "原始文件",
+  "none": "無",
+  "cancel": "取消",
+  "help": "聯繫支援",
+  "emailUs": "寄送電子郵件給我們",
+  "documentation": "文件",
+  "demo": [
+    {
+      "header": "了解 DocsGPT",
+      "query": "什麼是 DocsGPT？"
+    },
+    {
+      "header": "摘要文件",
+      "query": "摘要目前的內容"
+    },
+    {
+      "header": "撰寫程式碼",
+      "query": "為 /api/answer 撰寫 API 請求程式碼"
+    },
+    {
+      "header": "學習輔助",
+      "query": "為此內容撰寫可能的問題"
+    }
+  ],
+  "settings": {
+    "label": "設定",
+    "general": {
+      "label": "一般",
+      "selectTheme": "選擇主題",
+      "light": "淺色",
+      "dark": "深色",
+      "selectLanguage": "選擇語言",
+      "chunks": "每次查詢處理的區塊數",
+      "prompt": "使用中的提示",
+      "deleteAllLabel": "刪除所有對話",
+      "deleteAllBtn": "全部刪除",
+      "addNew": "新增",
+      "convHistory": "對話歷史記錄",
+      "none": "無",
+      "low": "低",
+      "medium": "中",
+      "high": "高",
+      "unlimited": "無限制",
+      "default": "預設"
+    },
+    "documents": {
+      "label": "文件",
+      "name": "文件名稱",
+      "date": "向量日期",
+      "type": "類型",
+      "tokenUsage": "Token 使用量"
+    },
+    "apiKeys": {
+      "label": "API 金鑰",
+      "name": "名稱",
+      "key": "API 金鑰",
+      "sourceDoc": "來源文件",
+      "createNew": "新增"
+    },
+    "analytics": {
+      "label": "分析"
+    },
+    "logs": {
+      "label": "日誌"
+    }
+  },
+  "modals": {
+    "uploadDoc": {
+      "label": "上傳新文件",
+      "file": "從檔案",
+      "remote": "遠端",
+      "name": "名稱",
+      "choose": "選擇檔案",
+      "info": "請上傳 .pdf, .txt, .rst, .docx, .md, .zip 檔案，大小限制為 25MB",
+      "uploadedFiles": "已上傳的檔案",
+      "cancel": "取消",
+      "train": "訓練",
+      "link": "連結",
+      "urlLink": "URL 連結",
+      "reddit": {
+        "id": "用戶端 ID",
+        "secret": "用戶端金鑰",
+        "agent": "使用者代理(User-Agent)",
+        "searchQueries": "搜尋查詢",
+        "numberOfPosts": "貼文數量"
+      }
+    },
+    "createAPIKey": {
+      "label": "建立新的 API 金鑰",
+      "apiKeyName": "API 金鑰名稱",
+      "chunks": "每次查詢處理的區塊數",
+      "prompt": "選擇使用中的提示",
+      "sourceDoc": "來源文件",
+      "create": "建立"
+    },
+    "saveKey": {
+      "note": "請儲存您的金鑰",
+      "disclaimer": "這是唯一一次顯示您的金鑰。",
+      "copy": "複製",
+      "copied": "已複製",
+      "confirm": "我已儲存金鑰"
+    },
+    "deleteConv": {
+      "confirm": "您確定要刪除所有對話嗎？",
+      "delete": "刪除"
+    },
+    "shareConv": {
+      "label": "建立公開頁面以分享",
+      "note": "來源文件、個人資訊和後續對話將保持私密",
+      "create": "建立"
+    }
+  },
+  "sharedConv": {
+    "subtitle": "使用以下工具建立",
+    "button": "開始使用 DocsGPT",
+    "meta": "DocsGPT 使用生成式 AI，請使用原始資料來源審閱重要資訊。"
+  },
+  "convTile": {
+    "share": "分享",
+    "delete": "刪除",
+    "rename": "重新命名",
+    "deleteWarning": "您確定要刪除這個對話嗎？"
+  }
+}

frontend/src/locale/zh.json

@@ -7,9 +7,12 @@
   "about": "关于",
   "inputPlaceholder": "在这里输入您的消息...",
   "tagline": "DocsGPT 使用 GenAI, 请使用来源审核关键信息.",
-  "sourceDocs": "来源文档",
+  "sourceDocs": "源",
   "none": "无",
   "cancel": "取消",
+  "help": "联系支持",
+  "emailUs": "给我们发邮件",
+  "documentation": "文档",
   "demo": [
     {
       "header": "了解 DocsGPT",
@@ -54,21 +57,33 @@
       "name": "文件名称",
       "date": "向量日期",
       "type": "类型",
-      "tokenUsage": "令牌使用"
+      "tokenUsage": "令牌使用",
+      "noData": "没有现有的文档"
     },
     "apiKeys": {
       "label": "聊天机器人",
       "name": "名称",
       "key": "API 密钥",
       "sourceDoc": "源文档",
-      "createNew": "创建新的"
+      "createNew": "创建新的",
+      "noData": "没有现有的聊天机器人"
+    },
+    "analytics": {
+      "label": "分析"
+    },
+    "logs": {
+      "label": "日志"
     }
   },
   "modals": {
     "uploadDoc": {
       "label": "上传新文档资料",
-      "file": "从文件",
-      "remote": "远程",
+      "select": "选择如何将文档上传到 DocsGPT",
+      "file": "从设备上传",
+      "back": "后退",
+      "wait": "请稍等 ...",
+      "remote": "从网站收集",
+      "start": "开始聊天",
       "name": "名称",
       "choose": "选择文件",
       "info": "请上传 .pdf, .txt, .rst, .docx, .md, .zip 文件，限 25MB",
@@ -77,6 +92,7 @@
       "train": "训练",
       "link": "链接",
       "urlLink": "URL 链接",
+      "repoUrl": "存储库 URL",
       "reddit": {
         "id": "客户端 ID",
         "secret": "客户端密钥",

frontend/src/modals/CreateAPIKeyModal.tsx

@@ -22,8 +22,9 @@ export default function CreateAPIKeyModal({
 
   const [APIKeyName, setAPIKeyName] = React.useState<string>('');
   const [sourcePath, setSourcePath] = React.useState<{
-    label: string;
-    value: string;
+    name: string;
+    id: string;
+    type: string;
   } | null>(null);
   const [prompt, setPrompt] = React.useState<{
     name: string;
@@ -41,27 +42,17 @@ export default function CreateAPIKeyModal({
       ? docs
           .filter((doc) => doc.model === embeddingsName)
           .map((doc: Doc) => {
-            let namePath = doc.name;
-            if (doc.language === namePath) {
-              namePath = '.project';
-            }
-            let docPath = 'default';
-            if (doc.location === 'local') {
-              docPath = 'local' + '/' + doc.name + '/';
-            } else if (doc.location === 'remote') {
-              docPath =
-                doc.language +
-                '/' +
-                namePath +
-                '/' +
-                doc.version +
-                '/' +
-                doc.model +
-                '/';
+            if ('id' in doc) {
+              return {
+                name: doc.name,
+                id: doc.id as string,
+                type: 'local',
+              };
             }
             return {
-              label: doc.name,
-              value: docPath,
+              name: doc.name,
+              id: doc.id ?? 'default',
+              type: doc.type ?? 'default',
             };
           })
       : [];
@@ -106,10 +97,14 @@ export default function CreateAPIKeyModal({
         <div className="my-4">
           <Dropdown
             placeholder={t('modals.createAPIKey.sourceDoc')}
-            selectedValue={sourcePath}
-            onSelect={(selection: { label: string; value: string }) =>
-              setSourcePath(selection)
-            }
+            selectedValue={sourcePath ? sourcePath.name : null}
+            onSelect={(selection: {
+              name: string;
+              id: string;
+              type: string;
+            }) => {
+              setSourcePath(selection);
+            }}
             options={extractDocPaths()}
             size="w-full"
             rounded="xl"
@@ -142,16 +137,22 @@ export default function CreateAPIKeyModal({
         </div>
         <button
           disabled={!sourcePath || APIKeyName.length === 0 || !prompt}
-          onClick={() =>
-            sourcePath &&
-            prompt &&
-            createAPIKey({
-              name: APIKeyName,
-              source: sourcePath.value,
-              prompt_id: prompt.id,
-              chunks: chunk,
-            })
-          }
+          onClick={() => {
+            if (sourcePath && prompt) {
+              const payload: any = {
+                name: APIKeyName,
+                prompt_id: prompt.id,
+                chunks: chunk,
+              };
+              if (sourcePath.type === 'default') {
+                payload.retriever = sourcePath.id;
+              }
+              if (sourcePath.type === 'local') {
+                payload.source = sourcePath.id;
+              }
+              createAPIKey(payload);
+            }
+          }}
           className="float-right mt-4 rounded-full bg-purple-30 px-5 py-2 text-sm text-white hover:bg-[#6F3FD1] disabled:opacity-50"
         >
           {t('modals.createAPIKey.create')}

frontend/src/modals/ShareConversationModal.tsx

@@ -46,27 +46,9 @@ export const ShareConversationModal = ({
       ? docs
           .filter((doc) => doc.model === embeddingsName)
           .map((doc: Doc) => {
-            let namePath = doc.name;
-            if (doc.language === namePath) {
-              namePath = '.project';
-            }
-            let docPath = 'default';
-            if (doc.location === 'local') {
-              docPath = 'local' + '/' + doc.name + '/';
-            } else if (doc.location === 'remote') {
-              docPath =
-                doc.language +
-                '/' +
-                namePath +
-                '/' +
-                doc.version +
-                '/' +
-                doc.model +
-                '/';
-            }
             return {
               label: doc.name,
-              value: docPath,
+              value: doc.id ?? 'default',
             };
           })
       : [];