parser functions change

token_func proposed change to chunking. open_ai_func proposed change to embedding_pipeline. Late chunking first  implementation requires further testing.

.github/dependabot.yml

@@ -13,3 +13,7 @@ updates:
     directory: "/frontend" # Location of package manifests
     schedule:
       interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/holopin.yml

@@ -1,5 +1,11 @@
-organization: arc53
-defaultSticker: clqmdf0ed34290glbvqh0kzxd
+organization: docsgpt
+defaultSticker: cm1ulwkkl180570cl82rtzympu
 stickers:
-  - id: clqmdf0ed34290glbvqh0kzxd
-    alias: festive
+  - id: cm1ulwkkl180570cl82rtzympu
+    alias: contributor2024
+  - id: cm1ureg8o130450cl8c1po6mil
+    alias: api
+  - id: cm1urhmag148240cl8yvqxkthx
+    alias: lpc
+  - id: cm1urlcpq622090cl2tvu4w71y
+    alias: lexeu

.github/labeler.yml

@@ -1,23 +1,31 @@
 repo:
-  - '*'
+- changed-files:
+  - any-glob-to-any-file: '*'
 
 github:
-  - .github/**/*
+- changed-files:
+  - any-glob-to-any-file: '.github/**/*'
 
 application:
-  - application/**/*
+- changed-files:
+  - any-glob-to-any-file: 'application/**/*'
 
 docs:
-  - docs/**/*
+- changed-files:
+  - any-glob-to-any-file: 'docs/**/*'
 
 extensions:
-  - extensions/**/*
+- changed-files:
+  - any-glob-to-any-file: 'extensions/**/*'
 
 frontend:
-  - frontend/**/*
+- changed-files:
+  - any-glob-to-any-file: 'frontend/**/*'
 
 scripts:
-  - scripts/**/*
+- changed-files:
+  - any-glob-to-any-file: 'scripts/**/*'
 
 tests:
-  - tests/**/*
+- changed-files:
+  - any-glob-to-any-file: 'tests/**/*'

.github/workflows/ci.yml

@@ -1,10 +1,8 @@
 name: Build and push DocsGPT Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
@@ -14,34 +12,36 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           file: './application/Dockerfile'
           platforms: linux/amd64
           context: ./application
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+          cache-to: type=inline

.github/workflows/cife.yml

@@ -1,10 +1,8 @@
 name: Build and push DocsGPT-FE Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
@@ -14,22 +12,22 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -37,12 +35,14 @@ jobs:
 
       # Runs a single command using the runners shell
       - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           file: './frontend/Dockerfile'
           platforms: linux/amd64, linux/arm64
           context: ./frontend
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+          cache-to: type=inline

.github/workflows/docker-develop-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v6
+        with:
+          file: './application/Dockerfile'
+          platforms: linux/amd64
+          context: ./application
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+          cache-to: type=inline

.github/workflows/docker-develop-fe-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT FE Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v6
+        with:
+          file: './frontend/Dockerfile'
+          platforms: linux/amd64
+          context: ./frontend
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+          cache-to: type=inline

.github/workflows/labeler.yml

@@ -10,7 +10,7 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/labeler@v4
+      - uses: actions/labeler@v5
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           sync-labels: true

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Lint with Ruff
         uses: chartboost/ruff-action@v1

.github/workflows/pytest.yml

@@ -8,9 +8,9 @@ jobs:
       matrix:
         python-version: ["3.11"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -24,7 +24,7 @@ jobs:
           python -m pytest --cov=application --cov-report=xml
       - name: Upload coverage reports to Codecov
         if: github.event_name == 'pull_request' && matrix.python-version == '3.11'
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/sync_fork.yaml

@@ -17,7 +17,7 @@ jobs:
     steps:
       # Step 1: run a standard checkout action
       - name: Checkout target repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Step 2: run the sync action
       - name: Sync upstream changes

CONTRIBUTING.md

@@ -6,7 +6,7 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 
 📣 **Discussions** - Engage in conversations, start new topics, or help answer questions.
 
-🐞 **Issues** - This is where we keep track of tasks. It could be bugs,fixes or suggestions for new features.
+🐞 **Issues** - This is where we keep track of tasks. It could be bugs, fixes or suggestions for new features.
 
 🛠️ **Pull requests** - Suggest changes to our repository, either by working on existing issues or adding new features.
 
@@ -21,8 +21,9 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 - If you're interested in contributing code, here are some important things to know:
 
 - We have a frontend built on React (Vite) and a backend in Python.
-=======
-Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version that you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
+
+  
+Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
 
 ### 👨‍💻 If you're interested in contributing code, here are some important things to know:
 
@@ -43,7 +44,7 @@ Please try to follow the guidelines.
 
 ### 🖥 If you are looking to contribute to Backend (🐍 Python):
 
-- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) or [`/scripts`](https://github.com/arc53/DocsGPT/tree/main/scripts) (please disregard old [`ingest_rst.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst.py) [`ingest_rst_sphinx.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst_sphinx.py) files; they will be deprecated soon).
+- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) or [`/scripts`](https://github.com/arc53/DocsGPT/tree/main/scripts) (please disregard old [`ingest_rst.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst.py) [`ingest_rst_sphinx.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst_sphinx.py) files; these will be deprecated soon).
 - All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
 - Before submitting your Pull Request, ensure it can be queried after ingesting some test data.
   
@@ -125,4 +126,4 @@ Thank you for considering contributing to DocsGPT! 🙏
 
 ## Questions/collaboration
 Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU). We're very friendly and welcoming to new contributors, so don't hesitate to reach out.
-# Thank you so much for considering to contribute DocsGPT!🙏
+# Thank you so much for considering to contributing DocsGPT!🙏

HACKTOBERFEST.md

@@ -4,10 +4,10 @@ Welcome, contributors! We're excited to announce that DocsGPT is participating i
 
 All contributors with accepted PRs will receive a cool Holopin! 🤩 (Watch out for a reply in your PR to collect it).
 
-### 🏆 Top 50 contributors will recieve a special T-shirt
+### 🏆 Top 50 contributors will receive a special T-shirt
 
 ### 🏆 [LLM Document analysis by LexEU competition](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md): 
-A separate competition is available for those sumbit best new retrieval / workflow method that will analyze a Document using EU laws.
+A separate competition is available for those who submit new retrieval / workflow method that will analyze a Document using EU laws.
 With 200$, 100$, 50$ prize for 1st, 2nd and 3rd place respectively.
 You can find more information [here](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md)
 
@@ -15,11 +15,15 @@ You can find more information [here](https://github.com/arc53/DocsGPT/blob/main/
 ```text
 🛠️ Code: This is the golden ticket! Make meaningful contributions through PRs.
 
-🧩 API extention: Build an app utilising DocsGPT API. We prefer submissions that showcase original ideas and turn the API into an AI agent.
+🧩 API extension: Build an app utilising DocsGPT API. We prefer submissions that showcase original ideas and turn the API into an AI agent.
+They can be a completely separate repos. 
+For example: 
+https://github.com/arc53/tg-bot-docsgpt-extenstion or 
+https://github.com/arc53/DocsGPT-cli
 
 Non-Code Contributions:
 
-📚 Wiki: Improve our documentation, Create a guide or change existing documentation.
+📚 Wiki: Improve our documentation, create a guide or change existing documentation.
 
 🖥️ Design: Improve the UI/UX or design a new feature.
 
@@ -33,5 +37,5 @@ Non-Code Contributions:
 - Refer to the [Documentation](https://docs.docsgpt.cloud/).
 - Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU) server. We're here to help newcomers, so don't hesitate to jump in! Join us [here](https://discord.gg/n5BX8dh8rU).
   
-Thank you very much for considering contributing to DocsGPT during Hacktoberfest! 🙏 Your contributions (not just simple typo) could earn you a stylish new t-shirt and other prizes as a token of our appreciation. 🎁 Join us, and let's code together! 🚀
+Thank you very much for considering contributing to DocsGPT during Hacktoberfest! 🙏 Your contributions (not just simple typos) could earn you a stylish new t-shirt and other prizes as a token of our appreciation. 🎁 Join us, and let's code together! 🚀
 

README.md

@@ -23,18 +23,16 @@ Say goodbye to time-consuming manual searches, and let <strong><a href="https://
  
 </div>
 
-### 🎃 [Hacktoberfest Prizes, Rules & Q&A](https://github.com/arc53/DocsGPT/blob/main/HACKTOBERFEST.md) 🎃
-
-### Our [Livestream to Dive into Hacktoberfest! Prizes, Rules & Q&A 🎉](https://www.youtube.com/watch?v=5QQaFFu9BC8) on 3rd of October 
-
 ### Production Support / Help for Companies:
 
 We're eager to provide personalized assistance when deploying your DocsGPT to a live environment.
 
-- [Book Enterprise / teams Demo :wave:](https://cal.com/arc53/docsgpt-demo-b2b?date=2024-09-27&month=2024-09)
-- [Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
+[Book a Meeting :wave:](https://cal.com/arc53/docsgpt-demo-b2b)⁠
 
-![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
+[Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
+
+
+<img src="https://github.com/user-attachments/assets/9a1f21de-7a15-4e42-9424-70d22ba5a913" alt="video-example-of-docs-gpt" width="1000" height="500">
 
 ## Roadmap
 
@@ -128,7 +126,7 @@ docker compose -f docker-compose-dev.yaml up -d
 > Make sure you have Python 3.10 or 3.11 installed.
 
 1. Export required environment variables or prepare a `.env` file in the project folder:
-   - Copy [.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) and create `.env`.
+   - Copy [.env-template](https://github.com/arc53/DocsGPT/blob/main/application/.env-template) and create `.env`.
 
 (check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.)
 
@@ -201,4 +199,9 @@ We as members, contributors, and leaders, pledge to make participation in our co
 
 The source code license is [MIT](https://opensource.org/license/mit/), as described in the [LICENSE](LICENSE) file.
 
-Built with [:bird: :link: LangChain](https://github.com/hwchase17/langchain)
+<p>This project is supported by:</p>
+<p>
+  <a href="https://www.digitalocean.com/?utm_medium=opensource&utm_source=DocsGPT">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px">
+  </a>
+</p>

application/api/answer/routes.py

@@ -11,8 +11,8 @@ from bson.objectid import ObjectId
 from flask import Blueprint, current_app, make_response, request, Response
 from flask_restx import fields, Namespace, Resource
 
-from pymongo import MongoClient
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 from application.error import bad_request
 from application.extensions import api
@@ -22,7 +22,7 @@ from application.utils import check_required_fields
 
 logger = logging.getLogger(__name__)
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
 sources_collection = db["sources"]
@@ -40,6 +40,8 @@ if settings.LLM_NAME == "openai":
     gpt_model = "gpt-3.5-turbo"
 elif settings.LLM_NAME == "anthropic":
     gpt_model = "claude-2"
+elif settings.LLM_NAME == "groq":
+    gpt_model = "llama3-8b-8192"
 
 if settings.MODEL_NAME:  # in case there is particular model name configured
     gpt_model = settings.MODEL_NAME
@@ -139,17 +141,17 @@ def save_conversation(conversation_id, question, response, source_log_docs, llm)
                 "role": "assistant",
                 "content": "Summarise following conversation in no more than 3 "
                 "words, respond ONLY with the summary, use the same "
-                "language as the system \n\nUser: "
-                + question
-                + "\n\n"
-                + "AI: "
-                + response,
+                "language as the system",
             },
             {
                 "role": "user",
                 "content": "Summarise following conversation in no more than 3 words, "
                 "respond ONLY with the summary, use the same language as the "
-                "system",
+                "system \n\nUser: "
+                + question
+                + "\n\n"
+                + "AI: "
+                + response,
             },
         ]
 
@@ -239,6 +241,7 @@ def complete_stream(
         yield f"data: {data}\n\n"
     except Exception as e:
         print("\033[91merr", str(e), file=sys.stderr)
+        traceback.print_exc()
         data = json.dumps(
             {
                 "type": "error",
@@ -267,9 +270,6 @@ class Stream(Resource):
             "prompt_id": fields.String(
                 required=False, default="default", description="Prompt ID"
             ),
-            "selectedDocs": fields.String(
-                required=False, description="Selected documents"
-            ),
             "chunks": fields.Integer(
                 required=False, default=2, description="Number of chunks"
             ),
@@ -290,6 +290,7 @@ class Stream(Resource):
     def post(self):
         data = request.get_json()
         required_fields = ["question"]
+
         missing_fields = check_required_fields(data, required_fields)
         if missing_fields:
             return missing_fields
@@ -300,10 +301,9 @@ class Stream(Resource):
             history = json.loads(history)
             conversation_id = data.get("conversation_id")
             prompt_id = data.get("prompt_id", "default")
-            if "selectedDocs" in data and data["selectedDocs"] is None:
-                chunks = 0
-            else:
-                chunks = int(data.get("chunks", 2))
+            
+
+            chunks = int(data.get("chunks", 2))
             token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
             retriever_name = data.get("retriever", "classic")
 
@@ -330,7 +330,8 @@ class Stream(Resource):
             )
 
             prompt = get_prompt(prompt_id)
-
+            if "isNoneDoc" in data and data["isNoneDoc"] is True:
+                chunks = 0
             retriever = RetrieverCreator.create_retriever(
                 retriever_name,
                 question=question,
@@ -420,7 +421,7 @@ class Answer(Resource):
     @api.doc(description="Provide an answer based on the question and retriever")
     def post(self):
         data = request.get_json()
-        required_fields = ["question"]
+        required_fields = ["question"]       
         missing_fields = check_required_fields(data, required_fields)
         if missing_fields:
             return missing_fields

application/api/internal/routes.py

@@ -1,13 +1,13 @@
 import os
 import datetime
 from flask import Blueprint, request, send_from_directory
-from pymongo import MongoClient
 from werkzeug.utils import secure_filename
 from bson.objectid import ObjectId
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
 sources_collection = db["sources"]
@@ -75,7 +75,7 @@ def upload_index_files():
                     "user": user,
                     "name": job_name,
                     "language": job_name,
-                    "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
+                    "date": datetime.datetime.now(),
                     "model": settings.EMBEDDINGS_NAME,
                     "type": type,
                     "tokens": tokens,
@@ -92,7 +92,7 @@ def upload_index_files():
                 "user": user,
                 "name": job_name,
                 "language": job_name,
-                "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
+                "date": datetime.datetime.now(),
                 "model": settings.EMBEDDINGS_NAME,
                 "type": type,
                 "tokens": tokens,

application/api/user/routes.py

@@ -2,23 +2,25 @@ import datetime
 import os
 import shutil
 import uuid
+import math
 
 from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
-from flask import Blueprint, jsonify, make_response, request
-from flask_restx import fields, Namespace, Resource
-from pymongo import MongoClient
+from flask import Blueprint, jsonify, make_response, request, redirect
+from flask_restx import inputs, fields, Namespace, Resource
 from werkzeug.utils import secure_filename
 
 from application.api.user.tasks import ingest, ingest_remote
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 from application.extensions import api
 from application.utils import check_required_fields
 from application.vectorstore.vector_creator import VectorCreator
+from application.tts.google_tts import GoogleTTS
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
 sources_collection = db["sources"]
@@ -314,7 +316,7 @@ class UploadFile(Resource):
                 for file in files:
                     filename = secure_filename(file.filename)
                     file.save(os.path.join(temp_dir, filename))
-
+                    print(f"Saved file: {filename}")
                 zip_path = shutil.make_archive(
                     base_name=os.path.join(save_dir, job_name),
                     format="zip",
@@ -322,6 +324,26 @@ class UploadFile(Resource):
                 )
                 final_filename = os.path.basename(zip_path)
                 shutil.rmtree(temp_dir)
+                task = ingest.delay(
+                    settings.UPLOAD_FOLDER,
+                    [
+                        ".rst",
+                        ".md",
+                        ".pdf",
+                        ".txt",
+                        ".docx",
+                        ".csv",
+                        ".epub",
+                        ".html",
+                        ".mdx",
+                        ".json",
+                        ".xlsx",
+                        ".pptx",
+                    ],
+                    job_name,
+                    final_filename,
+                    user,
+                )
             else:
                 file = files[0]
                 final_filename = secure_filename(file.filename)
@@ -340,14 +362,18 @@ class UploadFile(Resource):
                         ".epub",
                         ".html",
                         ".mdx",
+                        ".json",
+                        ".xlsx",
+                        ".pptx",
                     ],
                     job_name,
                     final_filename,
                     user,
                 )
-        except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
+        except Exception as err:
+            print(f"Error: {err}")
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
         return make_response(jsonify({"success": True, "task_id": task.id}), 200)
 
 
@@ -363,6 +389,7 @@ class UploadRemote(Resource):
                 ),
                 "name": fields.String(required=True, description="Job name"),
                 "data": fields.String(required=True, description="Data to process"),
+                "repo_url": fields.String(description="GitHub repository URL"),
             },
         )
     )
@@ -377,11 +404,18 @@ class UploadRemote(Resource):
             return missing_fields
 
         try:
+            if "repo_url" in data:
+                source_data = data["repo_url"]
+                loader = "github"
+            else:
+                source_data = data["data"]
+                loader = data["source"]
+
             task = ingest_remote.delay(
-                source_data=data["data"],
+                source_data=source_data,
                 job_name=data["name"],
                 user=data["user"],
-                loader=data["source"],
+                loader=loader,
             )
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
@@ -410,6 +444,11 @@ class TaskStatus(Resource):
 
             task = celery.AsyncResult(task_id)
             task_meta = task.info
+            print(f"Task status: {task.status}")
+            if not isinstance(
+                task_meta, (dict, list, str, int, float, bool, type(None))
+            ):
+                task_meta = str(task_meta)  # Convert to a string representation
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
@@ -417,6 +456,66 @@ class TaskStatus(Resource):
 
 
 @user_ns.route("/api/combine")
+class RedirectToSources(Resource):
+    @api.doc(
+        description="Redirects /api/combine to /api/sources for backward compatibility"
+    )
+    def get(self):
+        return redirect("/api/sources", code=301)
+
+
+@user_ns.route("/api/sources/paginated")
+class PaginatedSources(Resource):
+    @api.doc(description="Get document with pagination, sorting and filtering")
+    def get(self):
+        user = "local"
+        sort_field = request.args.get("sort", "date")  # Default to 'date'
+        sort_order = request.args.get("order", "desc")  # Default to 'desc'
+        page = int(request.args.get("page", 1))  # Default to 1
+        rows_per_page = int(request.args.get("rows", 10))  # Default to 10
+
+        # Prepare
+        query = {"user": user}
+        total_documents = sources_collection.count_documents(query)
+        total_pages = max(1, math.ceil(total_documents / rows_per_page))
+        sort_order = 1 if sort_order == "asc" else -1
+        skip = (page - 1) * rows_per_page
+
+        try:
+            documents = (
+                sources_collection.find(query)
+                .sort(sort_field, sort_order)
+                .skip(skip)
+                .limit(rows_per_page)
+            )
+
+            paginated_docs = []
+            for doc in documents:
+                doc_data = {
+                    "id": str(doc["_id"]),
+                    "name": doc.get("name", ""),
+                    "date": doc.get("date", ""),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "location": "local",
+                    "tokens": doc.get("tokens", ""),
+                    "retriever": doc.get("retriever", "classic"),
+                    "syncFrequency": doc.get("sync_frequency", ""),
+                }
+                paginated_docs.append(doc_data)
+
+            response = {
+                "total": total_documents,
+                "totalPages": total_pages,
+                "currentPage": page,
+                "paginated": paginated_docs,
+            }
+            return make_response(jsonify(response), 200)
+
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+
+@user_ns.route("/api/sources")
 class CombinedJson(Resource):
     @api.doc(description="Provide JSON file with combined available indexes")
     def get(self):
@@ -471,6 +570,7 @@ class CombinedJson(Resource):
                         "retriever": "brave_search",
                     }
                 )
+
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
@@ -794,7 +894,7 @@ class ShareConversation(Resource):
         if missing_fields:
             return missing_fields
 
-        is_promptable = request.args.get("isPromptable")
+        is_promptable = request.args.get("isPromptable", type=inputs.boolean)
         if is_promptable is None:
             return make_response(
                 jsonify({"success": False, "message": "isPromptable is required"}), 400
@@ -823,7 +923,7 @@ class ShareConversation(Resource):
                 uuid.uuid4(), UuidRepresentation.STANDARD
             )
 
-            if is_promptable.lower() == "true":
+            if is_promptable:
                 prompt_id = data.get("prompt_id", "default")
                 chunks = data.get("chunks", "2")
 
@@ -851,7 +951,7 @@ class ShareConversation(Resource):
                             "conversation_id": DBRef(
                                 "conversations", ObjectId(conversation_id)
                             ),
-                            "isPromptable": is_promptable.lower() == "true",
+                            "isPromptable": is_promptable,
                             "first_n_queries": current_n_queries,
                             "user": user,
                             "api_key": api_uuid,
@@ -875,7 +975,7 @@ class ShareConversation(Resource):
                                     "$ref": "conversations",
                                     "$id": ObjectId(conversation_id),
                                 },
-                                "isPromptable": is_promptable.lower() == "true",
+                                "isPromptable": is_promptable,
                                 "first_n_queries": current_n_queries,
                                 "user": user,
                                 "api_key": api_uuid,
@@ -910,7 +1010,7 @@ class ShareConversation(Resource):
                                 "$ref": "conversations",
                                 "$id": ObjectId(conversation_id),
                             },
-                            "isPromptable": is_promptable.lower() == "true",
+                            "isPromptable": is_promptable,
                             "first_n_queries": current_n_queries,
                             "user": user,
                             "api_key": api_uuid,
@@ -931,7 +1031,7 @@ class ShareConversation(Resource):
                     "conversation_id": DBRef(
                         "conversations", ObjectId(conversation_id)
                     ),
-                    "isPromptable": is_promptable.lower() == "false",
+                    "isPromptable": is_promptable,
                     "first_n_queries": current_n_queries,
                     "user": user,
                 }
@@ -954,7 +1054,7 @@ class ShareConversation(Resource):
                             "$ref": "conversations",
                             "$id": ObjectId(conversation_id),
                         },
-                        "isPromptable": is_promptable.lower() == "false",
+                        "isPromptable": is_promptable,
                         "first_n_queries": current_n_queries,
                         "user": user,
                     }
@@ -1653,3 +1753,36 @@ class ManageSync(Resource):
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
         return make_response(jsonify({"success": True}), 200)
+
+
+@user_ns.route("/api/tts")
+class TextToSpeech(Resource):
+    tts_model = api.model(
+        "TextToSpeechModel",
+        {
+            "text": fields.String(
+                required=True, description="Text to be synthesized as audio"
+            ),
+        },
+    )
+
+    @api.expect(tts_model)
+    @api.doc(description="Synthesize audio speech from text")
+    def post(self):
+        data = request.get_json()
+        text = data["text"]
+        try:
+            tts_instance = GoogleTTS()
+            audio_base64, detected_language = tts_instance.text_to_speech(text)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "audio_base64": audio_base64,
+                        "lang": detected_language,
+                    }
+                ),
+                200,
+            )
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)

application/cache.py

@@ -0,0 +1,93 @@
+import redis
+import time
+import json
+import logging
+from threading import Lock
+from application.core.settings import settings
+from application.utils import get_hash
+
+logger = logging.getLogger(__name__)
+
+_redis_instance = None
+_instance_lock = Lock()
+
+def get_redis_instance():
+    global _redis_instance
+    if _redis_instance is None:
+        with _instance_lock:
+            if _redis_instance is None:
+                try:
+                    _redis_instance = redis.Redis.from_url(settings.CACHE_REDIS_URL, socket_connect_timeout=2)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+                    _redis_instance = None
+    return _redis_instance
+
+def gen_cache_key(*messages, model="docgpt"):
+    if not all(isinstance(msg, dict) for msg in messages):
+        raise ValueError("All messages must be dictionaries.")
+    messages_str = json.dumps(list(messages), sort_keys=True)
+    combined = f"{model}_{messages_str}"
+    cache_key = get_hash(combined)
+    return cache_key
+
+def gen_cache(func):
+    def wrapper(self, model, messages, *args, **kwargs):
+        try:
+            cache_key = gen_cache_key(*messages)
+            redis_client = get_redis_instance()
+            if redis_client:
+                try:
+                    cached_response = redis_client.get(cache_key)
+                    if cached_response:
+                        return cached_response.decode('utf-8')
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            result = func(self, model, messages, *args, **kwargs)
+            if redis_client:
+                try:
+                    redis_client.set(cache_key, result, ex=1800)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            return result
+        except ValueError as e:
+            logger.error(e)
+            return "Error: No user message found in the conversation to generate a cache key."
+    return wrapper
+
+def stream_cache(func):
+    def wrapper(self, model, messages, stream, *args, **kwargs):
+        cache_key = gen_cache_key(*messages)
+        logger.info(f"Stream cache key: {cache_key}")
+        
+        redis_client = get_redis_instance()
+        if redis_client:
+            try:
+                cached_response = redis_client.get(cache_key)
+                if cached_response:
+                    logger.info(f"Cache hit for stream key: {cache_key}")
+                    cached_response = json.loads(cached_response.decode('utf-8'))
+                    for chunk in cached_response:
+                        yield chunk
+                        time.sleep(0.03)
+                    return
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+
+        result = func(self, model, messages, stream, *args, **kwargs)
+        stream_cache_data = []
+        
+        for chunk in result:
+            stream_cache_data.append(chunk)
+            yield chunk
+        
+        if redis_client:
+            try:
+                redis_client.set(cache_key, json.dumps(stream_cache_data), ex=1800)
+                logger.info(f"Stream cache saved for key: {cache_key}")
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+        
+    return wrapper

application/core/mongo_db.py

@@ -0,0 +1,24 @@
+from application.core.settings import settings
+from pymongo import MongoClient
+
+
+class MongoDB:
+    _client = None
+
+    @classmethod
+    def get_client(cls):
+        """
+        Get the MongoDB client instance, creating it if necessary.
+        """
+        if cls._client is None:
+            cls._client = MongoClient(settings.MONGO_URI)
+        return cls._client
+
+    @classmethod
+    def close_client(cls):
+        """
+        Close the MongoDB client connection.
+        """
+        if cls._client is not None:
+            cls._client.close()
+            cls._client = None

application/core/settings.py

@@ -18,9 +18,12 @@ class Settings(BaseSettings):
     DEFAULT_MAX_HISTORY: int = 150
     MODEL_TOKEN_LIMITS: dict = {"gpt-3.5-turbo": 4096, "claude-2": 1e5}
     UPLOAD_FOLDER: str = "inputs"
-    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant" or "milvus"
+    VECTOR_STORE: str = "faiss" #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb"
     RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
 
+    # LLM Cache
+    CACHE_REDIS_URL: str = "redis://localhost:6379/2"
+
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
 
     API_KEY: Optional[str] = None  # LLM api key
@@ -67,6 +70,9 @@ class Settings(BaseSettings):
     MILVUS_URI: Optional[str] = "./milvus_local.db"   # milvus lite version as default
     MILVUS_TOKEN: Optional[str] = ""
 
+    # LanceDB vectorstore config
+    LANCEDB_PATH: str = "/tmp/lancedb"  # Path where LanceDB stores its local data
+    LANCEDB_TABLE_NAME: Optional[str] = "docsgpts"  # Name of the table to use for storing vectors
     BRAVE_SEARCH_API_KEY: Optional[str] = None
 
     FLASK_DEBUG_MODE: bool = False

application/llm/base.py

@@ -1,28 +1,29 @@
 from abc import ABC, abstractmethod
 from application.usage import gen_token_usage, stream_token_usage
+from application.cache import stream_cache, gen_cache
 
 
 class BaseLLM(ABC):
     def __init__(self):
         self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
 
-    def _apply_decorator(self, method, decorator, *args, **kwargs):
-        return decorator(method, *args, **kwargs)
+    def _apply_decorator(self, method, decorators, *args, **kwargs):
+        for decorator in decorators:
+            method = decorator(method)
+        return method(self, *args, **kwargs)
 
     @abstractmethod
     def _raw_gen(self, model, messages, stream, *args, **kwargs):
         pass
 
     def gen(self, model, messages, stream=False, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen, gen_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
-        )
+        decorators = [gen_token_usage, gen_cache]
+        return self._apply_decorator(self._raw_gen, decorators=decorators, model=model, messages=messages, stream=stream, *args, **kwargs)
 
     @abstractmethod
     def _raw_gen_stream(self, model, messages, stream, *args, **kwargs):
         pass
 
     def gen_stream(self, model, messages, stream=True, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen_stream, stream_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
-        )
+        decorators = [stream_cache, stream_token_usage]
+        return self._apply_decorator(self._raw_gen_stream, decorators=decorators, model=model, messages=messages, stream=stream, *args, **kwargs)

application/llm/docsgpt_provider.py

@@ -9,35 +9,25 @@ class DocsGPTAPILLM(BaseLLM):
         super().__init__(*args, **kwargs)
         self.api_key = api_key
         self.user_api_key = user_api_key
-        self.endpoint = "https://llm.docsgpt.co.uk"
+        self.endpoint = "https://llm.arc53.com"
 
     def _raw_gen(self, baseself, model, messages, stream=False, *args, **kwargs):
-        context = messages[0]["content"]
-        user_question = messages[-1]["content"]
-        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
         response = requests.post(
-            f"{self.endpoint}/answer", json={"prompt": prompt, "max_new_tokens": 30}
+            f"{self.endpoint}/answer", json={"messages": messages, "max_new_tokens": 30}
         )
         response_clean = response.json()["a"].replace("###", "")
 
         return response_clean
 
     def _raw_gen_stream(self, baseself, model, messages, stream=True, *args, **kwargs):
-        context = messages[0]["content"]
-        user_question = messages[-1]["content"]
-        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
-        # send prompt to endpoint /stream
         response = requests.post(
             f"{self.endpoint}/stream",
-            json={"prompt": prompt, "max_new_tokens": 256},
+            json={"messages": messages, "max_new_tokens": 256},
             stream=True,
         )
 
         for line in response.iter_lines():
             if line:
-                # data = json.loads(line)
                 data_str = line.decode("utf-8")
                 if data_str.startswith("data: "):
                     data = json.loads(data_str[6:])

application/llm/google_ai.py

@@ -0,0 +1,48 @@
+from application.llm.base import BaseLLM
+
+class GoogleLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _clean_messages_google(self, messages):
+        return [
+            {
+                "role": "model" if message["role"] == "system" else message["role"],
+                "parts": [message["content"]],
+            }
+            for message in messages[1:]
+        ]
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        **kwargs
+    ):  
+        import google.generativeai as genai
+        genai.configure(api_key=self.api_key)
+        model = genai.GenerativeModel(model, system_instruction=messages[0]["content"])
+        response = model.generate_content(self._clean_messages_google(messages))
+        return response.text
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        **kwargs
+    ):  
+        import google.generativeai as genai
+        genai.configure(api_key=self.api_key)
+        model = genai.GenerativeModel(model, system_instruction=messages[0]["content"])
+        response = model.generate_content(self._clean_messages_google(messages), stream=True)
+        for line in response:
+            if line.text is not None:
+                yield line.text

application/llm/groq.py

@@ -0,0 +1,45 @@
+from application.llm.base import BaseLLM
+
+
+
+class GroqLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from openai import OpenAI
+
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        for line in response:
+            # import sys
+            # print(line.choices[0].delta.content, file=sys.stderr)
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content

application/llm/llm_creator.py

@@ -1,3 +1,4 @@
+from application.llm.groq import GroqLLM
 from application.llm.openai import OpenAILLM, AzureOpenAILLM
 from application.llm.sagemaker import SagemakerAPILLM
 from application.llm.huggingface import HuggingFaceLLM
@@ -5,6 +6,7 @@ from application.llm.llama_cpp import LlamaCpp
 from application.llm.anthropic import AnthropicLLM
 from application.llm.docsgpt_provider import DocsGPTAPILLM
 from application.llm.premai import PremAILLM
+from application.llm.google_ai import GoogleLLM
 
 
 class LLMCreator:
@@ -17,6 +19,8 @@ class LLMCreator:
         "anthropic": AnthropicLLM,
         "docsgpt": DocsGPTAPILLM,
         "premai": PremAILLM,
+        "groq": GroqLLM,
+        "google": GoogleLLM
     }
 
     @classmethod

application/parser/chunking.py

@@ -0,0 +1,118 @@
+import re
+from typing import List, Tuple, Union
+import logging
+from application.parser.schema.base import Document
+from application.utils import get_encoding
+
+logger = logging.getLogger(__name__)
+
+class Chunker:
+    def __init__(
+        self,
+        chunking_strategy: str = "classic_chunk",
+        max_tokens: int = 2000,
+        min_tokens: int = 150,
+        duplicate_headers: bool = False,
+    ):
+        if chunking_strategy not in ["classic_chunk"]:
+            raise ValueError(f"Unsupported chunking strategy: {chunking_strategy}")
+        self.chunking_strategy = chunking_strategy
+        self.max_tokens = max_tokens
+        self.min_tokens = min_tokens
+        self.duplicate_headers = duplicate_headers
+        self.encoding = get_encoding()
+
+    def separate_header_and_body(self, text: str) -> Tuple[str, str]:
+        header_pattern = r"^(.*?\n){3}"
+        match = re.match(header_pattern, text)
+        if match:
+            header = match.group(0)
+            body = text[len(header):]
+        else:
+            header, body = "", text  # No header, treat entire text as body
+        return header, body
+
+    def combine_documents(self, doc: Document, next_doc: Document) -> Document:
+        combined_text = doc.text + " " + next_doc.text
+        combined_token_count = len(self.encoding.encode(combined_text))
+        new_doc = Document(
+            text=combined_text,
+            doc_id=doc.doc_id,
+            embedding=doc.embedding,
+            extra_info={**(doc.extra_info or {}), "token_count": combined_token_count}
+        )
+        return new_doc
+    
+    def split_document(self, doc: Document) -> List[Document]:
+        split_docs = []
+        header, body = self.separate_header_and_body(doc.text)
+        header_tokens = self.encoding.encode(header) if header else []
+        body_tokens = self.encoding.encode(body)
+
+        current_position = 0
+        part_index = 0
+        while current_position < len(body_tokens):
+            end_position = current_position + self.max_tokens - len(header_tokens)
+            chunk_tokens = (header_tokens + body_tokens[current_position:end_position]
+                            if self.duplicate_headers or part_index == 0 else body_tokens[current_position:end_position])
+            chunk_text = self.encoding.decode(chunk_tokens)
+            new_doc = Document(
+                text=chunk_text,
+                doc_id=f"{doc.doc_id}-{part_index}",
+                embedding=doc.embedding,
+                extra_info={**(doc.extra_info or {}), "token_count": len(chunk_tokens)}
+            )
+            split_docs.append(new_doc)
+            current_position = end_position
+            part_index += 1
+            header_tokens = []
+        return split_docs
+
+    def classic_chunk(self, documents: List[Document]) -> List[Document]:
+        processed_docs = []
+        i = 0
+        while i < len(documents):
+            doc = documents[i]
+            tokens = self.encoding.encode(doc.text)
+            token_count = len(tokens)
+
+            if self.min_tokens <= token_count <= self.max_tokens:
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            elif token_count < self.min_tokens:
+                if i + 1 < len(documents):
+                    next_doc = documents[i + 1]
+                    next_tokens = self.encoding.encode(next_doc.text)
+                    if token_count + len(next_tokens) <= self.max_tokens:
+                        # Combine small documents
+                        combined_doc = self.combine_documents(doc, next_doc)
+                        processed_docs.append(combined_doc)
+                        i += 2
+                    else:
+                        # Keep the small document as is if adding next_doc would exceed max_tokens
+                        doc.extra_info = doc.extra_info or {}
+                        doc.extra_info["token_count"] = token_count
+                        processed_docs.append(doc)
+                        i += 1
+                else:
+                    # No next document to combine with; add the small document as is
+                    doc.extra_info = doc.extra_info or {}
+                    doc.extra_info["token_count"] = token_count
+                    processed_docs.append(doc)
+                    i += 1
+            else:
+                # Split large documents
+                processed_docs.extend(self.split_document(doc))
+                i += 1
+        return processed_docs
+
+    def chunk(
+        self,
+        documents: List[Document]
+    ) -> List[Document]:
+        if self.chunking_strategy == "classic_chunk":
+            return self.classic_chunk(documents)
+        else:
+            raise ValueError("Unsupported chunking strategy")

application/parser/embedding_pipeline.py

@@ -0,0 +1,86 @@
+import os
+import logging
+from retry import retry
+from tqdm import tqdm
+from application.core.settings import settings
+from application.vectorstore.vector_creator import VectorCreator
+
+
+@retry(tries=10, delay=60)
+def add_text_to_store_with_retry(store, doc, source_id):
+    """
+    Add a document's text and metadata to the vector store with retry logic.
+    Args:
+        store: The vector store object.
+        doc: The document to be added.
+        source_id: Unique identifier for the source.
+    """
+    try:
+        doc.metadata["source_id"] = str(source_id)
+        store.add_texts([doc.page_content], metadatas=[doc.metadata])
+    except Exception as e:
+        logging.error(f"Failed to add document with retry: {e}")
+        raise
+
+
+def embed_and_store_documents(docs, folder_name, source_id, task_status):
+    """
+    Embeds documents and stores them in a vector store.
+
+    Args:
+        docs (list): List of documents to be embedded and stored.
+        folder_name (str): Directory to save the vector store.
+        source_id (str): Unique identifier for the source.
+        task_status: Task state manager for progress updates.
+
+    Returns:
+        None
+    """
+    # Ensure the folder exists
+    if not os.path.exists(folder_name):
+        os.makedirs(folder_name)
+
+    # Initialize vector store
+    if settings.VECTOR_STORE == "faiss":
+        docs_init = [docs.pop(0)]
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            docs_init=docs_init,
+            source_id=folder_name,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+    else:
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            source_id=source_id,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+        store.delete_index()
+
+    total_docs = len(docs)
+
+    # Process and embed documents
+    for idx, doc in tqdm(
+        docs,
+        desc="Embedding 🦖",
+        unit="docs",
+        total=total_docs,
+        bar_format="{l_bar}{bar}| Time Left: {remaining}",
+    ):
+        try:
+            # Update task status for progress tracking
+            progress = int((idx / total_docs) * 100)
+            task_status.update_state(state="PROGRESS", meta={"current": progress})
+
+            # Add document to vector store
+            add_text_to_store_with_retry(store, doc, source_id)
+        except Exception as e:
+            logging.error(f"Error embedding document {idx}: {e}")
+            logging.info(f"Saving progress at document {idx} out of {total_docs}")
+            store.save_local(folder_name)
+            break
+
+    # Save the vector store
+    if settings.VECTOR_STORE == "faiss":
+        store.save_local(folder_name)
+    logging.info("Vector store saved successfully.")

application/parser/file/bulk.py

@@ -10,18 +10,23 @@ from application.parser.file.epub_parser import EpubParser
 from application.parser.file.html_parser import HTMLParser
 from application.parser.file.markdown_parser import MarkdownParser
 from application.parser.file.rst_parser import RstParser
-from application.parser.file.tabular_parser import PandasCSVParser
+from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser
+from application.parser.file.json_parser import JSONParser
+from application.parser.file.pptx_parser import PPTXParser
 from application.parser.schema.base import Document
 
 DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
     ".pdf": PDFParser(),
     ".docx": DocxParser(),
     ".csv": PandasCSVParser(),
+    ".xlsx":ExcelParser(),
     ".epub": EpubParser(),
     ".md": MarkdownParser(),
     ".rst": RstParser(),
     ".html": HTMLParser(),
     ".mdx": MarkdownParser(),
+    ".json":JSONParser(),
+    ".pptx":PPTXParser(),
 }
 
 

application/parser/file/json_parser.py

@@ -0,0 +1,57 @@
+import json
+from typing import Any, Dict, List, Union
+from pathlib import Path
+
+from application.parser.file.base_parser import BaseParser
+
+class JSONParser(BaseParser):
+    r"""JSON (.json) parser.
+
+    Parses JSON files into a list of strings or a concatenated document.
+    It handles both JSON objects (dictionaries) and arrays (lists).
+
+    Args:
+        concat_rows (bool): Whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each item in the JSON.
+            True by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        json_config (dict): Options for parsing JSON. Can be used to specify options like
+        custom decoding or formatting. Set to empty dict by default.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            row_joiner: str = "\n",
+            json_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._row_joiner = row_joiner
+        self._json_config = json_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse JSON file."""
+        
+        with open(file, 'r', encoding='utf-8') as f:
+                data = json.load(f, **self._json_config)
+
+        if isinstance(data, dict):
+            data = [data]
+
+        if self._concat_rows:
+            return self._row_joiner.join([str(item) for item in data])
+        else:
+            return data

application/parser/file/pptx_parser.py

@@ -0,0 +1,75 @@
+"""PPT parser.
+Contains parsers for presentation (.pptx) files to extract slide text.
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from application.parser.file.base_parser import BaseParser
+
+class PPTXParser(BaseParser):
+    r"""PPTX (.pptx) parser for extracting text from PowerPoint slides.
+    Args:
+        concat_slides (bool): Specifies whether to concatenate all slide text into one document.
+            - If True, slide texts will be joined together as a single string.
+            - If False, each slide's text will be stored as a separate entry in a list.
+            Set to True by default.
+        slide_separator (str): Separator used to join slides' text content.
+            Only used when `concat_slides=True`. Default is "\n".
+        Refer to https://python-pptx.readthedocs.io/en/latest/ for more information.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        concat_slides: bool = True,
+        slide_separator: str = "\n",
+        **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_slides = concat_slides
+        self._slide_separator = slide_separator
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        r"""
+        Parse a .pptx file and extract text from each slide.
+        Args:
+            file (Path): Path to the .pptx file.
+            errors (str): Error handling policy ('ignore' by default).
+        Returns:
+            Union[str, List[str]]: Concatenated text if concat_slides is True,
+            otherwise a list of slide texts.
+        """
+
+        try:
+            from pptx import Presentation
+        except ImportError:
+            raise ImportError("pptx module is required to read .PPTX files.")
+
+        try:
+            presentation = Presentation(file)
+            slide_texts=[]
+
+            # Iterate over each slide in the presentation
+            for slide in presentation.slides:
+                slide_text=""
+
+                # Iterate over each shape in the slide
+                for shape in slide.shapes:
+                    # Check if the shape has a 'text' attribute and append that to the slide_text
+                    if hasattr(shape,"text"):
+                        slide_text+=shape.text
+
+                slide_texts.append(slide_text.strip())
+
+            if self._concat_slides:
+                return self._slide_separator.join(slide_texts)
+            else:
+                return slide_texts
+
+        except Exception as e:
+            raise e

application/parser/file/tabular_parser.py

@@ -113,3 +113,68 @@ class PandasCSVParser(BaseParser):
             return (self._row_joiner).join(text_list)
         else:
             return text_list
+
+
+class ExcelParser(BaseParser):
+    r"""Excel (.xlsx) parser.
+
+    Parses Excel files using Pandas `read_excel` function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_excel` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the table structure on its own.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read Excel files.")
+
+        df = pd.read_excel(file, **self._pandas_config)
+
+        text_list = df.apply(
+            lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
+        ).tolist()
+
+        if self._concat_rows:
+            return (self._row_joiner).join(text_list)
+        else:
+            return text_list

application/parser/java2doc.py

@@ -1,66 +0,0 @@
-import os
-
-import javalang
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.java'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, "r") as file:
-        java_code = file.read()
-        methods = {}
-        tree = javalang.parse.parse(java_code)
-        for _, node in tree.filter(javalang.tree.MethodDeclaration):
-            method_name = node.name
-            start_line = node.position.line - 1
-            end_line = start_line
-            brace_count = 0
-            for line in java_code.splitlines()[start_line:]:
-                end_line += 1
-                brace_count += line.count("{") - line.count("}")
-                if brace_count == 0:
-                    break
-            method_source_code = "\n".join(java_code.splitlines()[start_line:end_line])
-            methods[method_name] = method_source_code
-    return methods
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = javalang.parse.parse(source_code)
-        for class_decl in tree.types:
-            class_name = class_decl.name
-            declarations = []
-            methods = []
-            for field_decl in class_decl.fields:
-                field_name = field_decl.declarators[0].name
-                field_type = field_decl.type.name
-                declarations.append(f"{field_type} {field_name}")
-            for method_decl in class_decl.methods:
-                methods.append(method_decl.name)
-            class_string = "Declarations: " + ", ".join(declarations) + "\n  Method name: " + ", ".join(methods)
-            classes[class_name] = class_string
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict

application/parser/js2doc.py

@@ -1,70 +0,0 @@
-import os
-
-import escodegen
-import esprima
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.js'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        functions = {}
-        tree = esprima.parseScript(source_code)
-        for node in tree.body:
-            if node.type == 'FunctionDeclaration':
-                func_name = node.id.name if node.id else '<anonymous>'
-                functions[func_name] = escodegen.generate(node)
-            elif node.type == 'VariableDeclaration':
-                for declaration in node.declarations:
-                    if declaration.init and declaration.init.type == 'FunctionExpression':
-                        func_name = declaration.id.name if declaration.id else '<anonymous>'
-                        functions[func_name] = escodegen.generate(declaration.init)
-            elif node.type == 'ClassDeclaration':
-                for subnode in node.body.body:
-                    if subnode.type == 'MethodDefinition':
-                        func_name = subnode.key.name
-                        functions[func_name] = escodegen.generate(subnode.value)
-                    elif subnode.type == 'VariableDeclaration':
-                        for declaration in subnode.declarations:
-                            if declaration.init and declaration.init.type == 'FunctionExpression':
-                                func_name = declaration.id.name if declaration.id else '<anonymous>'
-                                functions[func_name] = escodegen.generate(declaration.init)
-        return functions
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = esprima.parseScript(source_code)
-        for node in tree.body:
-            if node.type == 'ClassDeclaration':
-                class_name = node.id.name
-                function_names = []
-                for subnode in node.body.body:
-                    if subnode.type == 'MethodDefinition':
-                        function_names.append(subnode.key.name)
-                classes[class_name] = ", ".join(function_names)
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict

application/parser/late_chunking.py

@@ -0,0 +1,94 @@
+from typing import List, Tuple, Union, Optional
+from transformers import AutoTokenizer, AutoModel
+from sentence_transformers import SentenceTransformer
+import torch
+import torch.nn as nn
+from application.parser.schema.base import Document
+
+
+class LateChunker:
+    def __init__(self, model_name: str, late_tokens: int = 1000, **model_kwargs):
+        """
+        Initialize the LateChunker with a model, tokenizer, and late_tokens limit.
+        Supports both transformers and sentence-transformers models.
+        """
+        self.late_tokens = late_tokens
+        self.model_name = model_name
+
+        # Load model based on type
+        if "sentence-transformers" in model_name:
+            self.model = SentenceTransformer(model_name, **model_kwargs)
+            self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+            self.wrapper_type = "sentence_transformers"
+        else:
+            self.model = AutoModel.from_pretrained(model_name, trust_remote_code=True, **model_kwargs)
+            self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+            self.wrapper_type = "transformers"
+
+    def tokenize_with_offsets(self, text: str):
+        """Tokenize text and return tokens with character offsets."""
+        tokens = self.tokenizer.encode_plus(
+            text, return_offsets_mapping=True, add_special_tokens=False
+        )
+        return tokens["input_ids"], tokens["offset_mapping"]
+
+    def late_chunk_with_embeddings(
+        self, documents: List[Document]
+    ) -> List[Tuple[str, List[Tuple[int, int]], List[float]]]:
+        """
+        Combines documents into 'super chunks' that fit within `late_tokens` limit.
+        Outputs each super chunk with span annotations and embeddings.
+        """
+        super_chunks = []
+        current_super_chunk_text = []
+        current_token_count = 0
+        span_annotations = []
+
+        for doc in documents:
+            doc_text = doc.text
+            input_ids, offsets = self.tokenize_with_offsets(doc_text)
+            doc_token_count = len(input_ids)
+
+            # Check if adding this document exceeds the late_tokens limit
+            if current_token_count + doc_token_count > self.late_tokens:
+                # Finalize the current super chunk
+                combined_text = " ".join(current_super_chunk_text)
+                embeddings = self.generate_embeddings(combined_text)
+
+                super_chunks.append((combined_text, span_annotations, embeddings))
+
+                # Reset for a new super chunk
+                current_super_chunk_text = []
+                span_annotations = []
+                current_token_count = 0
+
+            # Add document to the current super chunk
+            start_token = current_token_count
+            end_token = current_token_count + doc_token_count
+            span_annotations.append((start_token, end_token))
+            current_super_chunk_text.append(doc_text)
+            current_token_count = end_token
+
+        # Add the final super chunk if there are remaining documents
+        if current_super_chunk_text:
+            combined_text = " ".join(current_super_chunk_text)
+            embeddings = self.generate_embeddings(combined_text)
+            super_chunks.append((combined_text, span_annotations, embeddings))
+
+        return super_chunks
+
+    def generate_embeddings(self, text: str) -> List[float]:
+        """Generate embeddings for a given text using the loaded model."""
+        if self.wrapper_type == "sentence_transformers":
+            # Sentence-Transformers
+            embeddings = self.model.encode([text])
+            return embeddings[0].tolist()
+
+        elif self.wrapper_type == "transformers":
+            # Transformers models
+            inputs = self.tokenizer(text, return_tensors="pt")
+            model_output = self.model(**inputs)
+            return model_output.last_hidden_state.mean(dim=1).squeeze().tolist()
+
+        else:
+            raise ValueError("Unsupported model type for embedding generation.")

application/parser/py2doc.py

@@ -1,121 +0,0 @@
-import ast
-import os
-from pathlib import Path
-
-import tiktoken
-from langchain.llms import OpenAI
-from langchain.prompts import PromptTemplate
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.py'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        functions = {}
-        tree = ast.parse(source_code)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.FunctionDef):
-                func_name = node.name
-                func_def = ast.get_source_segment(source_code, node)
-                functions[func_name] = func_def
-    return functions
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = ast.parse(source_code)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.ClassDef):
-                class_name = node.name
-                function_names = []
-                for subnode in ast.walk(node):
-                    if isinstance(subnode, ast.FunctionDef):
-                        function_names.append(subnode.name)
-                classes[class_name] = ", ".join(function_names)
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict
-
-
-def parse_functions(functions_dict, formats, dir):
-    c1 = len(functions_dict)
-    for i, (source, functions) in enumerate(functions_dict.items(), start=1):
-        print(f"Processing file {i}/{c1}")
-        source_w = source.replace(dir + "/", "").replace("." + formats, ".md")
-        subfolders = "/".join(source_w.split("/")[:-1])
-        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
-        for j, (name, function) in enumerate(functions.items(), start=1):
-            print(f"Processing function {j}/{len(functions)}")
-            prompt = PromptTemplate(
-                input_variables=["code"],
-                template="Code: \n{code}, \nDocumentation: ",
-            )
-            llm = OpenAI(temperature=0)
-            response = llm(prompt.format(code=function))
-            mode = "a" if Path(f"outputs/{source_w}").exists() else "w"
-            with open(f"outputs/{source_w}", mode) as f:
-                f.write(
-                    f"\n\n# Function name: {name} \n\nFunction: \n```\n{function}\n```, \nDocumentation: \n{response}")
-
-
-def parse_classes(classes_dict, formats, dir):
-    c1 = len(classes_dict)
-    for i, (source, classes) in enumerate(classes_dict.items()):
-        print(f"Processing file {i + 1}/{c1}")
-        source_w = source.replace(dir + "/", "").replace("." + formats, ".md")
-        subfolders = "/".join(source_w.split("/")[:-1])
-        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
-        for name, function_names in classes.items():
-            print(f"Processing Class {i + 1}/{c1}")
-            prompt = PromptTemplate(
-                input_variables=["class_name", "functions_names"],
-                template="Class name: {class_name} \nFunctions: {functions_names}, \nDocumentation: ",
-            )
-            llm = OpenAI(temperature=0)
-            response = llm(prompt.format(class_name=name, functions_names=function_names))
-
-            with open(f"outputs/{source_w}", "a" if Path(f"outputs/{source_w}").exists() else "w") as f:
-                f.write(f"\n\n# Class name: {name} \n\nFunctions: \n{function_names}, \nDocumentation: \n{response}")
-
-
-def transform_to_docs(functions_dict, classes_dict, formats, dir):
-    docs_content = ''.join([str(key) + str(value) for key, value in functions_dict.items()])
-    docs_content += ''.join([str(key) + str(value) for key, value in classes_dict.items()])
-
-    num_tokens = len(tiktoken.get_encoding("cl100k_base").encode(docs_content))
-    total_price = ((num_tokens / 1000) * 0.02)
-
-    print(f"Number of Tokens = {num_tokens:,d}")
-    print(f"Approx Cost = ${total_price:,.2f}")
-
-    user_input = input("Price Okay? (Y/N)\n").lower()
-    if user_input == "y" or user_input == "":
-        if not Path("outputs").exists():
-            Path("outputs").mkdir()
-        parse_functions(functions_dict, formats, dir)
-        parse_classes(classes_dict, formats, dir)
-        print("All done!")
-    else:
-        print("The API was not called. No money was spent.")

application/parser/remote/github_loader.py

@@ -0,0 +1,58 @@
+import base64
+import requests
+from typing import List
+from application.parser.remote.base import BaseRemote
+from langchain_core.documents import Document
+import mimetypes
+
+class GitHubLoader(BaseRemote):
+    def __init__(self):
+        self.access_token = None
+        self.headers = {
+            "Authorization": f"token {self.access_token}"
+        } if self.access_token else {}
+        return
+
+    def fetch_file_content(self, repo_url: str, file_path: str) -> str:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
+        response = requests.get(url, headers=self.headers)
+
+        if response.status_code == 200:
+            content = response.json()
+            mime_type, _ = mimetypes.guess_type(file_path)  # Guess the MIME type based on the file extension
+
+            if content.get("encoding") == "base64":
+                if mime_type and mime_type.startswith("text"):  # Handle only text files
+                    try:
+                        decoded_content = base64.b64decode(content["content"]).decode("utf-8")
+                        return f"Filename: {file_path}\n\n{decoded_content}"
+                    except Exception as e:
+                        raise e
+                else:
+                    return f"Filename: {file_path} is a binary file and was skipped."
+            else:
+                return f"Filename: {file_path}\n\n{content['content']}"
+        else:
+            response.raise_for_status()
+
+    def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
+        response = requests.get(url, headers={**self.headers, "Accept": "application/vnd.github.v3.raw"})
+        contents = response.json()
+        files = []
+        for item in contents:
+            if item["type"] == "file":
+                files.append(item["path"])
+            elif item["type"] == "dir":
+                files.extend(self.fetch_repo_files(repo_url, item["path"]))
+        return files
+
+    def load_data(self, repo_url: str) -> List[Document]:
+        repo_name = repo_url.split("github.com/")[-1]
+        files = self.fetch_repo_files(repo_name)
+        documents = []
+        for file_path in files:
+            content = self.fetch_file_content(repo_name, file_path)
+            documents.append(Document(page_content=content, metadata={"title": file_path, 
+            "source": f"https://github.com/{repo_name}/blob/main/{file_path}"}))
+        return documents

application/parser/remote/reddit_loader.py

@@ -1,10 +1,19 @@
 from application.parser.remote.base import BaseRemote
 from langchain_community.document_loaders import RedditPostsLoader
+import json
 
 
 class RedditPostsLoaderRemote(BaseRemote):
     def load_data(self, inputs):
-        data = eval(inputs)
+        try:
+            data = json.loads(inputs)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON input: {e}")
+
+        required_fields = ["client_id", "client_secret", "user_agent", "search_queries"]
+        missing_fields = [field for field in required_fields if field not in data]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
         client_id = data.get("client_id")
         client_secret = data.get("client_secret")
         user_agent = data.get("user_agent")

application/parser/remote/remote_creator.py

@@ -2,6 +2,7 @@ from application.parser.remote.sitemap_loader import SitemapLoader
 from application.parser.remote.crawler_loader import CrawlerLoader
 from application.parser.remote.web_loader import WebLoader
 from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
+from application.parser.remote.github_loader import GitHubLoader
 
 
 class RemoteCreator:
@@ -10,6 +11,7 @@ class RemoteCreator:
         "sitemap": SitemapLoader,
         "crawler": CrawlerLoader,
         "reddit": RedditPostsLoaderRemote,
+        "github": GitHubLoader,
     }
 
     @classmethod

application/requirements.txt

@@ -4,7 +4,7 @@ beautifulsoup4==4.12.3
 celery==5.3.6
 dataclasses-json==0.6.7
 docx2txt==0.8
-duckduckgo-search==6.2.6
+duckduckgo-search==6.3.0
 ebooklib==0.18
 elastic-transport==8.15.0
 elasticsearch==8.15.1
@@ -14,6 +14,7 @@ esutils==1.0.1
 Flask==3.0.3
 faiss-cpu==1.8.0.post1
 flask-restx==1.3.0
+gTTS==2.3.2
 gunicorn==23.0.0
 html2text==2024.2.26
 javalang==0.13.0
@@ -49,11 +50,12 @@ openapi3-parser==1.1.18
 orjson==3.10.7
 packaging==24.1
 pandas==2.2.3
+openpyxl==3.1.5
 pathable==0.4.3
 pillow==10.4.0
 portalocker==2.10.1
 prance==23.6.21.0
-primp==0.6.2
+primp==0.6.3
 prompt-toolkit==3.0.47
 protobuf==5.28.2
 py==1.11.0
@@ -64,6 +66,7 @@ pymongo==4.8.0
 pypdf2==3.0.1
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
+python-pptx==1.0.2
 qdrant-client==1.11.0
 redis==5.0.1
 referencing==0.30.2
@@ -83,4 +86,4 @@ urllib3==2.2.3
 vine==5.1.0
 wcwidth==0.2.13
 werkzeug==3.0.4
-yarl==1.11.1
+yarl==1.11.1

application/retriever/brave_search.py

@@ -75,7 +75,6 @@ class BraveRetSearch(BaseRetriever):
         if len(self.chat_history) > 1:
             tokens_current_history = 0
             # count tokens in history
-            self.chat_history.reverse()
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
                     tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(

application/retriever/classic_rag.py

@@ -45,7 +45,6 @@ class ClassicRAG(BaseRetriever):
                 settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
             )
             docs_temp = docsearch.search(self.question, k=self.chunks)
-            print(docs_temp)
             docs = [
                 {
                     "title": i.metadata.get(
@@ -60,8 +59,6 @@ class ClassicRAG(BaseRetriever):
                 }
                 for i in docs_temp
             ]
-        if settings.LLM_NAME == "llama.cpp":
-            docs = [docs[0]]
 
         return docs
 
@@ -78,7 +75,6 @@ class ClassicRAG(BaseRetriever):
         if len(self.chat_history) > 1:
             tokens_current_history = 0
             # count tokens in history
-            self.chat_history.reverse()
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
                     tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
@@ -97,7 +93,6 @@ class ClassicRAG(BaseRetriever):
         llm = LLMCreator.create_llm(
             settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
         )
-
         completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
         for line in completion:
             yield {"answer": str(line)}

application/retriever/duckduck_search.py

@@ -92,7 +92,6 @@ class DuckDuckSearch(BaseRetriever):
         if len(self.chat_history) > 1:
             tokens_current_history = 0
             # count tokens in history
-            self.chat_history.reverse()
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
                     tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(

application/tts/base.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+
+
+class BaseTTS(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def text_to_speech(self, *args, **kwargs):
+        pass

application/tts/elevenlabs.py

@@ -0,0 +1,29 @@
+from io import BytesIO
+import base64
+from application.tts.base import BaseTTS
+
+
+class ElevenlabsTTS(BaseTTS):
+    def __init__(self):
+        from elevenlabs.client import ElevenLabs
+
+        self.client = ElevenLabs(
+            api_key="ELEVENLABS_API_KEY",
+            )
+    
+
+    def text_to_speech(self, text):
+        lang = "en"
+        audio = self.client.generate(
+            text=text,
+            model="eleven_multilingual_v2",
+            voice="Brian",
+        )
+        audio_data = BytesIO()
+        for chunk in audio:
+            audio_data.write(chunk)
+        audio_bytes = audio_data.getvalue()
+
+        # Encode to base64
+        audio_base64 = base64.b64encode(audio_bytes).decode("utf-8")
+        return audio_base64, lang

application/tts/google_tts.py

@@ -0,0 +1,19 @@
+import io
+import base64
+from gtts import gTTS
+from application.tts.base import BaseTTS
+
+
+class GoogleTTS(BaseTTS):
+    def __init__(self):
+        pass
+
+
+    def text_to_speech(self, text):
+        lang = "en"
+        audio_fp = io.BytesIO()
+        tts = gTTS(text=text, lang=lang, slow=False)
+        tts.write_to_fp(audio_fp)
+        audio_fp.seek(0)
+        audio_base64 = base64.b64encode(audio_fp.read()).decode("utf-8")
+        return audio_base64, lang

application/usage.py

@@ -1,10 +1,9 @@
 import sys
-from pymongo import MongoClient
 from datetime import datetime
-from application.core.settings import settings
+from application.core.mongo_db import MongoDB
 from application.utils import num_tokens_from_string
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 usage_collection = db["token_usage"]
 

application/utils.py

@@ -1,6 +1,8 @@
 import tiktoken
+import hashlib
 from flask import jsonify, make_response
 
+
 _encoding = None
 
 
@@ -39,3 +41,8 @@ def check_required_fields(data, required_fields):
             400,
         )
     return None
+
+
+def get_hash(data):
+    return hashlib.md5(data.encode()).hexdigest()
+

application/vectorstore/faiss.py

@@ -3,30 +3,27 @@ from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
 import os
 
-def get_vectorstore(path):
+def get_vectorstore(path: str) -> str:
     if path:
-        vectorstore = "indexes/"+path
-        vectorstore = os.path.join("application", vectorstore)
+        vectorstore = os.path.join("application", "indexes", path)
     else:
         vectorstore = os.path.join("application")
-
     return vectorstore
 
 class FaissStore(BaseVectorStore):
-
-    def __init__(self, source_id, embeddings_key, docs_init=None):
+    def __init__(self, source_id: str, embeddings_key: str, docs_init=None):
         super().__init__()
         self.path = get_vectorstore(source_id)
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
-        if docs_init:
-            self.docsearch = FAISS.from_documents(
-                docs_init, embeddings
-            )
-        else:
-            self.docsearch = FAISS.load_local(
-                self.path, embeddings, 
-                allow_dangerous_deserialization=True
-            )
+
+        try:
+            if docs_init:
+                self.docsearch = FAISS.from_documents(docs_init, embeddings)
+            else:
+                self.docsearch = FAISS.load_local(self.path, embeddings, allow_dangerous_deserialization=True)
+        except Exception:
+            raise
+
         self.assert_embedding_dimensions(embeddings)
 
     def search(self, *args, **kwargs):
@@ -42,16 +39,12 @@ class FaissStore(BaseVectorStore):
         return self.docsearch.delete(*args, **kwargs)
 
     def assert_embedding_dimensions(self, embeddings):
-        """
-        Check that the word embedding dimension of the docsearch index matches
-        the dimension of the word embeddings used 
-        """
+        """Check that the word embedding dimension of the docsearch index matches the dimension of the word embeddings used."""
         if settings.EMBEDDINGS_NAME == "huggingface_sentence-transformers/all-mpnet-base-v2":
-            try:
-                word_embedding_dimension = embeddings.dimension
-            except AttributeError as e:
-                raise AttributeError("'dimension' attribute not found in embeddings instance. Make sure the embeddings object is properly initialized.") from e
+            word_embedding_dimension = getattr(embeddings, 'dimension', None)
+            if word_embedding_dimension is None:
+                raise AttributeError("'dimension' attribute not found in embeddings instance.")
+            
             docsearch_index_dimension = self.docsearch.index.d
             if word_embedding_dimension != docsearch_index_dimension:
-                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) " +
-                                 f"!= docsearch index dimension ({docsearch_index_dimension})")
+                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) != docsearch index dimension ({docsearch_index_dimension})")

application/vectorstore/lancedb.py

@@ -0,0 +1,119 @@
+from typing import List, Optional
+import importlib
+from application.vectorstore.base import BaseVectorStore
+from application.core.settings import settings
+
+class LanceDBVectorStore(BaseVectorStore):
+    """Class for LanceDB Vector Store integration."""
+
+    def __init__(self, path: str = settings.LANCEDB_PATH,
+                 table_name_prefix: str = settings.LANCEDB_TABLE_NAME,
+                 source_id: str = None,
+                 embeddings_key: str = "embeddings"):
+        """Initialize the LanceDB vector store."""
+        super().__init__()
+        self.path = path
+        self.table_name = f"{table_name_prefix}_{source_id}" if source_id else table_name_prefix
+        self.embeddings_key = embeddings_key
+        self._lance_db = None
+        self.docsearch = None
+        self._pa = None  # PyArrow (pa) will be lazy loaded
+
+    @property
+    def pa(self):
+        """Lazy load pyarrow module."""
+        if self._pa is None:
+            self._pa = importlib.import_module("pyarrow")
+        return self._pa
+
+    @property
+    def lancedb(self):
+        """Lazy load lancedb module."""
+        if not hasattr(self, "_lancedb_module"):
+            self._lancedb_module = importlib.import_module("lancedb")
+        return self._lancedb_module
+
+    @property
+    def lance_db(self):
+        """Lazy load the LanceDB connection."""
+        if self._lance_db is None:
+            self._lance_db = self.lancedb.connect(self.path)
+        return self._lance_db
+
+    @property
+    def table(self):
+        """Lazy load the LanceDB table."""
+        if self.docsearch is None:
+            if self.table_name in self.lance_db.table_names():
+                self.docsearch = self.lance_db.open_table(self.table_name)
+            else:
+                self.docsearch = None
+        return self.docsearch
+
+    def ensure_table_exists(self):
+        """Ensure the table exists before performing operations."""
+        if self.table is None:
+            embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
+            schema = self.pa.schema([
+                self.pa.field("vector", self.pa.list_(self.pa.float32(), list_size=embeddings.dimension)),
+                self.pa.field("text", self.pa.string()),
+                self.pa.field("metadata", self.pa.struct([
+                    self.pa.field("key", self.pa.string()),
+                    self.pa.field("value", self.pa.string())
+                ]))
+            ])
+            self.docsearch = self.lance_db.create_table(self.table_name, schema=schema)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]] = None, source_id: str = None):
+        """Add texts with metadata and their embeddings to the LanceDB table."""
+        embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_documents(texts)
+        vectors = []
+        for embedding, text, metadata in zip(embeddings, texts, metadatas or [{}] * len(texts)):
+            if source_id:
+                metadata["source_id"] = source_id
+            metadata_struct = [{"key": k, "value": str(v)} for k, v in metadata.items()]
+            vectors.append({
+                "vector": embedding,
+                "text": text,
+                "metadata": metadata_struct
+            })
+        self.ensure_table_exists()
+        self.docsearch.add(vectors)
+
+    def search(self, query: str, k: int = 2, *args, **kwargs):
+        """Search LanceDB for the top k most similar vectors."""
+        self.ensure_table_exists()
+        query_embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_query(query)
+        results = self.docsearch.search(query_embedding).limit(k).to_list()
+        return [(result["_distance"], result["text"], result["metadata"]) for result in results]
+
+    def delete_index(self):
+        """Delete the entire LanceDB index (table)."""
+        if self.table:
+            self.lance_db.drop_table(self.table_name)
+
+    def assert_embedding_dimensions(self, embeddings):
+        """Ensure that embedding dimensions match the table index dimensions."""
+        word_embedding_dimension = embeddings.dimension
+        if self.table:
+            table_index_dimension = len(self.docsearch.schema["vector"].type.value_type)
+            if word_embedding_dimension != table_index_dimension:
+                raise ValueError(
+                    f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) "
+                    f"!= table index dimension ({table_index_dimension})"
+                )
+
+    def filter_documents(self, filter_condition: dict) -> List[dict]:
+        """Filter documents based on certain conditions."""
+        self.ensure_table_exists()
+
+        # Ensure source_id exists in the filter condition
+        if 'source_id' not in filter_condition:
+            raise ValueError("filter_condition must contain 'source_id'")
+
+        source_id = filter_condition["source_id"]
+
+        # Use LanceDB's native filtering if supported, otherwise filter manually
+        filtered_data = self.docsearch.filter(lambda x: x.metadata and x.metadata.get("source_id") == source_id).to_list()
+
+        return filtered_data

application/vectorstore/milvus.py

@@ -7,7 +7,7 @@ from application.vectorstore.base import BaseVectorStore
 
 
 class MilvusStore(BaseVectorStore):
-    def __init__(self, path: str = "", embeddings_key: str = "embeddings"):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
         super().__init__()
         from langchain_milvus import Milvus
 
@@ -20,10 +20,11 @@ class MilvusStore(BaseVectorStore):
             collection_name=settings.MILVUS_COLLECTION_NAME,
             connection_args=connection_args,
         )
-        self._path = path
+        self._source_id = source_id
 
     def search(self, question, k=2, *args, **kwargs):
-        return self._docsearch.similarity_search(query=question, k=k, filter={"path": self._path} *args, **kwargs)
+        expr = f"source_id == '{self._source_id}'"
+        return self._docsearch.similarity_search(query=question, k=k, expr=expr, *args, **kwargs)
 
     def add_texts(self, texts: List[str], metadatas: Optional[List[dict]], *args, **kwargs):
         ids = [str(uuid4()) for _ in range(len(texts))]

application/worker.py

@@ -8,8 +8,8 @@ from urllib.parse import urljoin
 
 import requests
 from bson.objectid import ObjectId
-from pymongo import MongoClient
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 from application.parser.file.bulk import SimpleDirectoryReader
 from application.parser.open_ai_func import call_openai_api
@@ -18,26 +18,27 @@ from application.parser.schema.base import Document
 from application.parser.token_func import group_split
 from application.utils import count_tokens_docs
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 sources_collection = db["sources"]
 
+# Constants
+MIN_TOKENS = 150
+MAX_TOKENS = 1250
+RECURSION_DEPTH = 2
 
 # Define a function to extract metadata from a given filename.
 def metadata_from_filename(title):
     return {"title": title}
 
-
 # Define a function to generate a random string of a given length.
 def generate_random_string(length):
     return "".join([string.ascii_letters[i % 52] for i in range(length)])
 
-
 current_dir = os.path.dirname(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 )
 
-
 def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
     """
     Recursively extract zip files with a limit on recursion depth.
@@ -52,9 +53,13 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
         logging.warning(f"Reached maximum recursion depth of {max_depth}")
         return
 
-    with zipfile.ZipFile(zip_path, "r") as zip_ref:
-        zip_ref.extractall(extract_to)
-    os.remove(zip_path)  # Remove the zip file after extracting
+    try:
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(extract_to)
+        os.remove(zip_path)  # Remove the zip file after extracting
+    except Exception as e:
+        logging.error(f"Error extracting zip file {zip_path}: {e}")
+        return
 
     # Check for nested zip files and extract them
     for root, dirs, files in os.walk(extract_to):
@@ -64,6 +69,38 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
                 file_path = os.path.join(root, file)
                 extract_zip_recursive(file_path, root, current_depth + 1, max_depth)
 
+def download_file(url, params, dest_path):
+    try:
+        response = requests.get(url, params=params)
+        response.raise_for_status()
+        with open(dest_path, "wb") as f:
+            f.write(response.content)
+    except requests.RequestException as e:
+        logging.error(f"Error downloading file: {e}")
+        raise
+
+def upload_index(full_path, file_data):
+    try:
+        if settings.VECTOR_STORE == "faiss":
+            files = {
+                "file_faiss": open(full_path + "/index.faiss", "rb"),
+                "file_pkl": open(full_path + "/index.pkl", "rb"),
+            }
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
+            )
+        else:
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), data=file_data
+            )
+        response.raise_for_status()
+    except requests.RequestException as e:
+        logging.error(f"Error uploading index: {e}")
+        raise
+    finally:
+        if settings.VECTOR_STORE == "faiss":
+            for file in files.values():
+                file.close()
 
 # Define the main function for ingesting and processing documents.
 def ingest_worker(
@@ -84,39 +121,25 @@ def ingest_worker(
     Returns:
         dict: Information about the completed ingestion task, including input parameters and a "limited" flag.
     """
-    # directory = 'inputs' or 'temp'
-    # formats = [".rst", ".md"]
     input_files = None
     recursive = True
     limit = None
     exclude = True
-    # name_job = 'job1'
-    # filename = 'install.rst'
-    # user = 'local'
     sample = False
     token_check = True
-    min_tokens = 150
-    max_tokens = 1250
-    recursion_depth = 2
     full_path = os.path.join(directory, user, name_job)
 
     logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job})
-    # check if API_URL env variable is set
     file_data = {"name": name_job, "file": filename, "user": user}
-    response = requests.get(
-        urljoin(settings.API_URL, "/api/download"), params=file_data
-    )
-    file = response.content
 
     if not os.path.exists(full_path):
         os.makedirs(full_path)
-    with open(os.path.join(full_path, filename), "wb") as f:
-        f.write(file)
+    download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename))
 
     # check if file is .zip and extract it
     if filename.endswith(".zip"):
         extract_zip_recursive(
-            os.path.join(full_path, filename), full_path, 0, recursion_depth
+            os.path.join(full_path, filename), full_path, 0, RECURSION_DEPTH
         )
 
     self.update_state(state="PROGRESS", meta={"current": 1})
@@ -132,8 +155,8 @@ def ingest_worker(
     ).load_data()
     raw_docs = group_split(
         documents=raw_docs,
-        min_tokens=min_tokens,
-        max_tokens=max_tokens,
+        min_tokens=MIN_TOKENS,
+        max_tokens=MAX_TOKENS,
         token_check=token_check,
     )
 
@@ -148,28 +171,13 @@ def ingest_worker(
         for i in range(min(5, len(raw_docs))):
             logging.info(f"Sample document {i}: {raw_docs[i]}")
 
-    # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
-    # and send them to the server (provide user and name in form)
-    file_data = {
-        "name": name_job,
-        "user": user,
+    file_data.update({
         "tokens": tokens,
         "retriever": retriever,
         "id": str(id),
         "type": "local",
-    }
-    if settings.VECTOR_STORE == "faiss":
-        files = {
-            "file_faiss": open(full_path + "/index.faiss", "rb"),
-            "file_pkl": open(full_path + "/index.pkl", "rb"),
-        }
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
-        )
-    else:
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), data=file_data
-        )
+    })
+    upload_index(full_path, file_data)
 
     # delete local
     shutil.rmtree(full_path)
@@ -183,7 +191,6 @@ def ingest_worker(
         "limited": False,
     }
 
-
 def remote_worker(
     self,
     source_data,
@@ -197,16 +204,14 @@ def remote_worker(
     doc_id=None,
 ):
     token_check = True
-    min_tokens = 150
-    max_tokens = 1250
-    full_path = directory + "/" + user + "/" + name_job
+    full_path = os.path.join(directory, user, name_job)
 
     if not os.path.exists(full_path):
         os.makedirs(full_path)
     self.update_state(state="PROGRESS", meta={"current": 1})
     logging.info(
         f"Remote job: {full_path}",
-        extra={"user": user, "job": name_job, source_data: source_data},
+        extra={"user": user, "job": name_job, "source_data": source_data},
     )
 
     remote_loader = RemoteCreator.create_loader(loader)
@@ -214,11 +219,10 @@ def remote_worker(
 
     docs = group_split(
         documents=raw_docs,
-        min_tokens=min_tokens,
-        max_tokens=max_tokens,
+        min_tokens=MIN_TOKENS,
+        max_tokens=MAX_TOKENS,
         token_check=token_check,
     )
-    # docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
     tokens = count_tokens_docs(docs)
     if operation_mode == "upload":
         id = ObjectId()
@@ -230,7 +234,6 @@ def remote_worker(
         call_openai_api(docs, full_path, id, self)
     self.update_state(state="PROGRESS", meta={"current": 100})
 
-    # Proceed with uploading and cleaning as in the original function
     file_data = {
         "name": name_job,
         "user": user,
@@ -241,23 +244,12 @@ def remote_worker(
         "remote_data": source_data,
         "sync_frequency": sync_frequency,
     }
-    if settings.VECTOR_STORE == "faiss":
-        files = {
-            "file_faiss": open(full_path + "/index.faiss", "rb"),
-            "file_pkl": open(full_path + "/index.pkl", "rb"),
-        }
-
-        requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
-        )
-    else:
-        requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
+    upload_index(full_path, file_data)
 
     shutil.rmtree(full_path)
 
     return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}
 
-
 def sync(
     self,
     source_data,
@@ -283,10 +275,10 @@ def sync(
             doc_id,
         )
     except Exception as e:
+        logging.error(f"Error during sync: {e}")
         return {"status": "error", "error": str(e)}
     return {"status": "success"}
 
-
 def sync_worker(self, frequency):
     sync_counts = Counter()
     sources = sources_collection.find()

docker-compose.yaml

@@ -20,6 +20,7 @@ services:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - CACHE_REDIS_URL=redis://redis:6379/2
     ports:
       - "7091:7091"
     volumes:
@@ -41,6 +42,7 @@ services:
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
+      - CACHE_REDIS_URL=redis://redis:6379/2
     depends_on:
       - redis
       - mongo

docs/README.md

@@ -46,6 +46,6 @@ yarn install
 yarn dev
 ```
 
-- Now, you should be able to view the docs on your local environment by visiting `http://localhost:5000`. You can explore the different markdown files and make changes as you see fit.
+- Now, you should be able to view the docs on your local environment by visiting `http://localhost:3000`. You can explore the different markdown files and make changes as you see fit.
 
 - **Footnotes:** This guide assumes you have Node.js and npm installed. The guide involves running a local server using yarn, and viewing the documentation offline. If you encounter any issues, it may be worth verifying your Node.js and npm installations and whether you have installed yarn correctly.

docs/package-lock.json

@@ -7,7 +7,7 @@
       "license": "MIT",
       "dependencies": {
         "@vercel/analytics": "^1.1.1",
-        "docsgpt": "^0.4.1",
+        "docsgpt": "^0.4.7",
         "next": "^14.2.12",
         "nextra": "^2.13.2",
         "nextra-theme-docs": "^2.13.2",
@@ -422,11 +422,6 @@
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@bpmn-io/snarkdown": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/@bpmn-io/snarkdown/-/snarkdown-2.2.0.tgz",
-      "integrity": "sha512-bVD7FIoaBDZeCJkMRgnBPDeptPlto87wt2qaCjf5t8iLaevDmTPaREd6FpBEGsHlUdHFFZWRk4qAoEC5So2M0Q=="
-    },
     "node_modules/@braintree/sanitize-url": {
       "version": "6.0.4",
       "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-6.0.4.tgz",
@@ -1175,58 +1170,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/@parcel/core": {
-      "version": "2.12.0",
-      "resolved": "https://registry.npmjs.org/@parcel/core/-/core-2.12.0.tgz",
-      "integrity": "sha512-s+6pwEj+GfKf7vqGUzN9iSEPueUssCCQrCBUlcAfKrJe0a22hTUCjewpB0I7lNrCIULt8dkndD+sMdOrXsRl6Q==",
-      "peer": true,
-      "dependencies": {
-        "@mischnic/json-sourcemap": "^0.1.0",
-        "@parcel/cache": "2.12.0",
-        "@parcel/diagnostic": "2.12.0",
-        "@parcel/events": "2.12.0",
-        "@parcel/fs": "2.12.0",
-        "@parcel/graph": "3.2.0",
-        "@parcel/logger": "2.12.0",
-        "@parcel/package-manager": "2.12.0",
-        "@parcel/plugin": "2.12.0",
-        "@parcel/profiler": "2.12.0",
-        "@parcel/rust": "2.12.0",
-        "@parcel/source-map": "^2.1.1",
-        "@parcel/types": "2.12.0",
-        "@parcel/utils": "2.12.0",
-        "@parcel/workers": "2.12.0",
-        "abortcontroller-polyfill": "^1.1.9",
-        "base-x": "^3.0.8",
-        "browserslist": "^4.6.6",
-        "clone": "^2.1.1",
-        "dotenv": "^7.0.0",
-        "dotenv-expand": "^5.1.0",
-        "json5": "^2.2.0",
-        "msgpackr": "^1.9.9",
-        "nullthrows": "^1.1.1",
-        "semver": "^7.5.2"
-      },
-      "engines": {
-        "node": ">= 12.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/parcel"
-      }
-    },
-    "node_modules/@parcel/core/node_modules/semver": {
-      "version": "7.6.3",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
-      "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
-      "peer": true,
-      "bin": {
-        "semver": "bin/semver.js"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
     "node_modules/@parcel/diagnostic": {
       "version": "2.12.0",
       "resolved": "https://registry.npmjs.org/@parcel/diagnostic/-/diagnostic-2.12.0.tgz",
@@ -1277,22 +1220,6 @@
         "@parcel/core": "^2.12.0"
       }
     },
-    "node_modules/@parcel/graph": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/@parcel/graph/-/graph-3.2.0.tgz",
-      "integrity": "sha512-xlrmCPqy58D4Fg5umV7bpwDx5Vyt7MlnQPxW68vae5+BA4GSWetfZt+Cs5dtotMG2oCHzZxhIPt7YZ7NRyQzLA==",
-      "peer": true,
-      "dependencies": {
-        "nullthrows": "^1.1.1"
-      },
-      "engines": {
-        "node": ">= 12.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/parcel"
-      }
-    },
     "node_modules/@parcel/logger": {
       "version": "2.12.0",
       "resolved": "https://registry.npmjs.org/@parcel/logger/-/logger-2.12.0.tgz",
@@ -2717,12 +2644,6 @@
         "server-only": "^0.0.1"
       }
     },
-    "node_modules/abortcontroller-polyfill": {
-      "version": "1.7.5",
-      "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.5.tgz",
-      "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ==",
-      "peer": true
-    },
     "node_modules/acorn": {
       "version": "8.11.3",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
@@ -2802,15 +2723,6 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
-    "node_modules/base-x": {
-      "version": "3.0.10",
-      "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.10.tgz",
-      "integrity": "sha512-7d0s06rR9rYaIWHkpfLIFICM/tkSVdoPC9qYAQRpxn9DdKNWNsKC0uk++akckyLq16Tx2WIinnZ6WRriAt6njQ==",
-      "peer": true,
-      "dependencies": {
-        "safe-buffer": "^5.0.1"
-      }
-    },
     "node_modules/boolbase": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
@@ -3025,15 +2937,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/clone": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
-      "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==",
-      "peer": true,
-      "engines": {
-        "node": ">=0.8"
-      }
-    },
     "node_modules/clsx": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.0.tgz",
@@ -3162,30 +3065,6 @@
         "cytoscape": "^3.2.0"
       }
     },
-    "node_modules/cytoscape-fcose": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz",
-      "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==",
-      "dependencies": {
-        "cose-base": "^2.2.0"
-      },
-      "peerDependencies": {
-        "cytoscape": "^3.2.0"
-      }
-    },
-    "node_modules/cytoscape-fcose/node_modules/cose-base": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz",
-      "integrity": "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==",
-      "dependencies": {
-        "layout-base": "^2.0.0"
-      }
-    },
-    "node_modules/cytoscape-fcose/node_modules/layout-base": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz",
-      "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg=="
-    },
     "node_modules/d3": {
       "version": "7.8.5",
       "resolved": "https://registry.npmjs.org/d3/-/d3-7.8.5.tgz",
@@ -3697,13 +3576,12 @@
       }
     },
     "node_modules/docsgpt": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/docsgpt/-/docsgpt-0.4.1.tgz",
-      "integrity": "sha512-9oH638vIg8I+zsjLV5Rp21yYniAtiTcyuBSByqWl2KoBdF/8vDSmr491l8n+ikbaTLiCW4uRU0p0r3BvRizy2Q==",
+      "version": "0.4.7",
+      "resolved": "https://registry.npmjs.org/docsgpt/-/docsgpt-0.4.7.tgz",
+      "integrity": "sha512-4YZzLZo6ybudFrJVUQflDFeWzFiTATRWB9myrGSpLigyuMMzax1ZAY2xFallZLuEG9VVm0mOgkx3ssWHLrXWkQ==",
       "license": "Apache-2.0",
       "dependencies": {
         "@babel/plugin-transform-flow-strip-types": "^7.23.3",
-        "@bpmn-io/snarkdown": "^2.2.0",
         "@parcel/resolver-glob": "^2.12.0",
         "@parcel/transformer-svg-react": "^2.12.0",
         "@parcel/transformer-typescript-tsc": "^2.12.0",
@@ -3715,6 +3593,7 @@
         "flow-bin": "^0.229.2",
         "i": "^0.3.7",
         "install": "^0.13.0",
+        "markdown-it": "^14.1.0",
         "npm": "^10.5.0",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
@@ -3786,30 +3665,15 @@
         "url": "https://github.com/fb55/domutils?sponsor=1"
       }
     },
-    "node_modules/dotenv": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-7.0.0.tgz",
-      "integrity": "sha512-M3NhsLbV1i6HuGzBUH8vXrtxOk+tWmzWKDMbAVSUp3Zsjm7ywFeuwrUXhmhQyRK1q5B5GGy7hcXPbj3bnfZg2g==",
-      "peer": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
-    "node_modules/dotenv-expand": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz",
-      "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==",
-      "peer": true
-    },
     "node_modules/electron-to-chromium": {
       "version": "1.4.693",
       "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.693.tgz",
       "integrity": "sha512-/if4Ueg0GUQlhCrW2ZlXwDAm40ipuKo+OgeHInlL8sbjt+hzISxZK949fZeJaVsheamrzANXvw1zQTvbxTvSHw=="
     },
     "node_modules/elkjs": {
-      "version": "0.8.2",
-      "resolved": "https://registry.npmjs.org/elkjs/-/elkjs-0.8.2.tgz",
-      "integrity": "sha512-L6uRgvZTH+4OF5NE/MBbzQx/WYpru1xCBE9respNj6qznEewGUIfhzmm7horWWxbNO2M0WckQypGctR8lH79xQ=="
+      "version": "0.9.3",
+      "resolved": "https://registry.npmjs.org/elkjs/-/elkjs-0.9.3.tgz",
+      "integrity": "sha512-f/ZeWvW/BCXbhGEf1Ujp29EASo/lk1FDnETgNKwJrsVvGZhUWCZyg3xLJjAsxfOmt8KjswHmI5EwCQcPMpOYhQ=="
     },
     "node_modules/entities": {
       "version": "4.5.0",
@@ -4859,6 +4723,15 @@
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
       "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
     },
+    "node_modules/linkify-it": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz",
+      "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==",
+      "license": "MIT",
+      "dependencies": {
+        "uc.micro": "^2.0.0"
+      }
+    },
     "node_modules/lmdb": {
       "version": "2.8.5",
       "resolved": "https://registry.npmjs.org/lmdb/-/lmdb-2.8.5.tgz",
@@ -4944,6 +4817,29 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/markdown-it": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz",
+      "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==",
+      "license": "MIT",
+      "dependencies": {
+        "argparse": "^2.0.1",
+        "entities": "^4.4.0",
+        "linkify-it": "^5.0.0",
+        "mdurl": "^2.0.0",
+        "punycode.js": "^2.3.1",
+        "uc.micro": "^2.1.0"
+      },
+      "bin": {
+        "markdown-it": "bin/markdown-it.mjs"
+      }
+    },
+    "node_modules/markdown-it/node_modules/argparse": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
+      "license": "Python-2.0"
+    },
     "node_modules/markdown-table": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.3.tgz",
@@ -5487,23 +5383,29 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/mdurl": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
+      "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==",
+      "license": "MIT"
+    },
     "node_modules/mermaid": {
-      "version": "10.6.1",
-      "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.6.1.tgz",
-      "integrity": "sha512-Hky0/RpOw/1il9X8AvzOEChfJtVvmXm+y7JML5C//ePYMy0/9jCEmW1E1g86x9oDfW9+iVEdTV/i+M6KWRNs4A==",
+      "version": "10.9.3",
+      "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.9.3.tgz",
+      "integrity": "sha512-V80X1isSEvAewIL3xhmz/rVmc27CVljcsbWxkxlWJWY/1kQa4XOABqpDl2qQLGKzpKm6WbTfUEKImBlUfFYArw==",
       "dependencies": {
         "@braintree/sanitize-url": "^6.0.1",
         "@types/d3-scale": "^4.0.3",
         "@types/d3-scale-chromatic": "^3.0.0",
-        "cytoscape": "^3.23.0",
+        "cytoscape": "^3.28.1",
         "cytoscape-cose-bilkent": "^4.1.0",
-        "cytoscape-fcose": "^2.1.0",
         "d3": "^7.4.0",
         "d3-sankey": "^0.12.3",
         "dagre-d3-es": "7.0.10",
         "dayjs": "^1.11.7",
-        "dompurify": "^3.0.5",
-        "elkjs": "^0.8.2",
+        "dompurify": "^3.0.5 <3.1.7",
+        "elkjs": "^0.9.0",
+        "katex": "^0.16.9",
         "khroma": "^2.0.0",
         "lodash-es": "^4.17.21",
         "mdast-util-from-markdown": "^1.3.0",
@@ -9338,6 +9240,15 @@
       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
       "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ=="
     },
+    "node_modules/punycode.js": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",
+      "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/react": {
       "version": "18.2.0",
       "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
@@ -9687,26 +9598,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/safe-buffer": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
-      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "peer": true
-    },
     "node_modules/safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
@@ -10051,18 +9942,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/typescript": {
-      "version": "5.6.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz",
-      "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==",
-      "peer": true,
-      "bin": {
-        "tsc": "bin/tsc",
-        "tsserver": "bin/tsserver"
-      },
-      "engines": {
-        "node": ">=14.17"
-      }
+    "node_modules/uc.micro": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz",
+      "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
+      "license": "MIT"
     },
     "node_modules/unified": {
       "version": "10.1.2",

docs/package.json

@@ -7,7 +7,7 @@
   "license": "MIT",
   "dependencies": {
     "@vercel/analytics": "^1.1.1",
-    "docsgpt": "^0.4.1",
+    "docsgpt": "^0.4.7",
     "next": "^14.2.12",
     "nextra": "^2.13.2",
     "nextra-theme-docs": "^2.13.2",

docs/pages/Guides/How-to-train-on-other-documentation.mdx

@@ -28,15 +28,15 @@ Navigate to the sidebar where you will find `Source Docs` option,here you will f
 
 
 ### Step 2
-Click on the `Upload icon` just beside the source docs options,now borwse and upload the document which you want to train on or select the `remote` option if you have to insert the link of the documentation.
+Click on the `Upload icon` just beside the source docs options,now browse and upload the document which you want to train on or select the `remote` option if you have to insert the link of the documentation.
 
 
 ### Step 3
-Now you will be able to see the name of the file uploaded under the Uploaded Files ,now click on `Train`,once you click on train it might take some time to train on the document. You will be able to see the `Training progress` and once the training is completed you can click the `finish` button and there you go your docuemnt is uploaded.
+Now you will be able to see the name of the file uploaded under the Uploaded Files ,now click on `Train`,once you click on train it might take some time to train on the document. You will be able to see the `Training progress` and once the training is completed you can click the `finish` button and there you go your document is uploaded.
 
 
 ### Step 4
-Go to `New chat`  and from the side bar select the document  you uploaded  under the `Source Docs` and go ahead with your chat, now you can ask qestions regarding the document you uploaded and you will get the effective answer based on it.
+Go to `New chat`  and from the side bar select the document  you uploaded  under the `Source Docs` and go ahead with your chat, now you can ask questions regarding the document you uploaded and you will get the effective answer based on it.
 
 </Steps>
 

docs/pages/Guides/How-to-use-different-LLM.mdx

@@ -33,7 +33,7 @@ For open source you  have to edit .env file with LLM_NAME with their desired LLM
 All the supported LLM providers are here application/llm and you  can check what env variable are needed for each
 List of latest supported LLMs are https://github.com/arc53/DocsGPT/blob/main/application/llm/llm_creator.py
 ### Step 3
-Visit application/llm and select the file of your selected llm and there you will find the speicifc requirements needed to be filled in order to use it,i.e API key of that llm.
+Visit application/llm and select the file of your selected llm and there you will find the specific requirements needed to be filled in order to use it,i.e API key of that llm.
 </Steps>
 
 ### For OpenAI-Compatible Endpoints:

docs/pages/_app.mdx

@@ -4,7 +4,7 @@ export default function MyApp({ Component, pageProps }) {
   return (
     <>
       <Component {...pageProps} />
-        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" theme="dark" />
+        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" theme="dark" size="medium" />
     </>
   )
 }

docs/theme.config.jsx

@@ -51,6 +51,9 @@ const config = {
   footer: {
     text: `MIT ${new Date().getFullYear()} © DocsGPT`,
   },
+  editLink: {
+    content: 'Edit this page on GitHub',
+  },
   logo() {
     return (
       <div className="flex items-center gap-2">

extensions/discord/bot.py

@@ -1,25 +1,60 @@
 import os
 import re
-
+import logging
+import aiohttp
 import discord
-import requests
 from discord.ext import commands
 import dotenv
 
 dotenv.load_dotenv()
 
-# Replace 'YOUR_BOT_TOKEN' with your bot's token
+# Enable logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Bot configuration
 TOKEN = os.getenv("DISCORD_TOKEN")
-PREFIX = '@DocsGPT'
-BASE_API_URL = 'http://localhost:7091'
+PREFIX = '!'  # Command prefix
+BASE_API_URL = os.getenv("API_BASE", "https://gptcloud.arc53.com")
+API_URL = BASE_API_URL + "/api/answer"
+API_KEY = os.getenv("API_KEY")
 
 intents = discord.Intents.default()
 intents.message_content = True
 
 bot = commands.Bot(command_prefix=PREFIX, intents=intents)
 
+# Store conversation history per user
+conversation_histories = {}
+
+def chunk_string(text, max_length=2000):
+    """Splits a string into chunks of a specified maximum length."""
+    # Create list to store the split strings
+    chunks = []
+    # Loop through the text, create substrings with max_length
+    while len(text) > max_length:
+        # Find last space within the limit
+        idx = text.rfind(' ', 0, max_length)
+        # Ensure we don't have an empty part
+        if idx == -1:
+            # If no spaces, just take chunk
+            chunks.append(text[:max_length])
+            text = text[max_length:]
+        else:
+            # Push whatever we've got up to the last space
+            chunks.append(text[:idx])
+            text = text[idx+1:]
+    # Catches the remaining part
+    chunks.append(text)
+    return chunks
+
+def escape_markdown(text):
+    """Escapes Discord markdown characters."""
+    escape_chars = r'\*_$$$$()~>#+-=|{}.!'
+    return re.sub(f'([{re.escape(escape_chars)}])', r'\\\1', text)
 
 def split_string(input_str):
+    """Splits the input string to detect bot mentions."""
     pattern = r'^<@!?{0}>\s*'.format(bot.user.id)
     match = re.match(pattern, input_str)
     if match:
@@ -27,42 +62,97 @@ def split_string(input_str):
         return str(bot.user.id), content
     return None, input_str
 
-
 @bot.event
 async def on_ready():
     print(f'{bot.user.name} has connected to Discord!')
 
-
-async def fetch_answer(question):
-    data = {
-        'sender': 'discord',
-        'question': question,
-        'history': ''
+async def generate_answer(question, messages, conversation_id):
+    """Generates an answer using the external API."""
+    payload = {
+        "question": question,
+        "api_key": API_KEY,
+        "history": messages,
+        "conversation_id": conversation_id
     }
-    headers = {"Content-Type": "application/json",
-               "Accept": "application/json"}
-    response = requests.post(BASE_API_URL + '/api/answer', json=data, headers=headers)
-    if response.status_code == 200:
-        return response.json()['answer']
-    return 'Sorry, I could not fetch the answer.'
+    headers = {
+        "Content-Type": "application/json; charset=utf-8"
+    }
+    timeout = aiohttp.ClientTimeout(total=60)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        async with session.post(API_URL, json=payload, headers=headers) as resp:
+            if resp.status == 200:
+                data = await resp.json()
+                conversation_id = data.get("conversation_id")
+                answer = data.get("answer", "Sorry, I couldn't find an answer.")
+                return {"answer": answer, "conversation_id": conversation_id}
+            else:
+                return {"answer": "Sorry, I couldn't find an answer.", "conversation_id": None}
 
+@bot.command(name="start")
+async def start(ctx):
+    """Handles the /start command."""
+    await ctx.send(f"Hi {ctx.author.mention}! How can I assist you today?")
+
+@bot.command(name="custom_help")
+async def custom_help_command(ctx):
+    """Handles the /custom_help command."""
+    help_text = (
+        "Here are the available commands:\n"
+        "`!start` - Begin a new conversation with the bot\n"
+        "`!help` - Display this help message\n\n"
+        "You can also mention me or send a direct message to ask a question!"
+    )
+    await ctx.send(help_text)
 
 @bot.event
 async def on_message(message):
     if message.author == bot.user:
         return
 
-    content = message.content.strip()
-    prefix, content = split_string(content)
-    if prefix is None:
-        return
-
-    part_prefix = str(bot.user.id)
-    if part_prefix == prefix:
-        answer = await fetch_answer(content)
-        await message.channel.send(answer)
-
+    # Process commands first
     await bot.process_commands(message)
 
+    # Check if the message is in a DM channel
+    if isinstance(message.channel, discord.DMChannel):
+        content = message.content.strip()
+    else:
+        # In guild channels, check if the message mentions the bot at the start
+        content = message.content.strip()
+        prefix, content = split_string(content)
+        if prefix is None:
+            return
+        part_prefix = str(bot.user.id)
+        if part_prefix != prefix:
+            return  # Bot not mentioned at the start, so do not process
 
-bot.run(TOKEN)
+    # Now process the message
+    user_id = message.author.id
+    if user_id not in conversation_histories:
+        conversation_histories[user_id] = {
+            "history": [],
+            "conversation_id": None
+        }
+
+    conversation = conversation_histories[user_id]
+    conversation["history"].append({"prompt": content})
+
+    # Generate the answer
+    response_doc = await generate_answer(
+        content,
+        conversation["history"],
+        conversation["conversation_id"]
+    )
+    answer = response_doc["answer"]
+    conversation_id = response_doc["conversation_id"]
+
+    answer_chunks = chunk_string(answer)
+    for chunk in answer_chunks:
+        await message.channel.send(chunk)
+
+    conversation["history"][-1]["response"] = answer
+    conversation["conversation_id"] = conversation_id
+
+    # Keep conversation history to last 10 exchanges
+    conversation["history"] = conversation["history"][-10:]
+
+bot.run(TOKEN)

extensions/react-widget/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "docsgpt",
-  "version": "0.4.2",
+  "version": "0.4.7",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "docsgpt",
-      "version": "0.4.2",
+      "version": "0.4.7",
       "license": "Apache-2.0",
       "dependencies": {
         "@babel/plugin-transform-flow-strip-types": "^7.23.3",
@@ -1885,17 +1885,6 @@
         "node": ">=6.0.0"
       }
     },
-    "node_modules/@jridgewell/source-map": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz",
-      "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@jridgewell/gen-mapping": "^0.3.5",
-        "@jridgewell/trace-mapping": "^0.3.25"
-      }
-    },
     "node_modules/@jridgewell/sourcemap-codec": {
       "version": "1.4.15",
       "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
@@ -2269,6 +2258,7 @@
       "version": "2.12.0",
       "resolved": "https://registry.npmjs.org/@parcel/core/-/core-2.12.0.tgz",
       "integrity": "sha512-s+6pwEj+GfKf7vqGUzN9iSEPueUssCCQrCBUlcAfKrJe0a22hTUCjewpB0I7lNrCIULt8dkndD+sMdOrXsRl6Q==",
+      "dev": true,
       "dependencies": {
         "@mischnic/json-sourcemap": "^0.1.0",
         "@parcel/cache": "2.12.0",
@@ -2308,6 +2298,7 @@
       "version": "7.6.2",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz",
       "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==",
+      "dev": true,
       "bin": {
         "semver": "bin/semver.js"
       },
@@ -2369,6 +2360,7 @@
       "version": "3.2.0",
       "resolved": "https://registry.npmjs.org/@parcel/graph/-/graph-3.2.0.tgz",
       "integrity": "sha512-xlrmCPqy58D4Fg5umV7bpwDx5Vyt7MlnQPxW68vae5+BA4GSWetfZt+Cs5dtotMG2oCHzZxhIPt7YZ7NRyQzLA==",
+      "dev": true,
       "dependencies": {
         "nullthrows": "^1.1.1"
       },
@@ -4568,7 +4560,7 @@
       "version": "0.5.11",
       "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.11.tgz",
       "integrity": "sha512-YNlnKRWF2sVojTpIyzwou9XoTNbzbzONwRhOoniEioF1AtaitTvVZblaQRrAzChWQ1bLYyYSWzM18y4WwgzJ+A==",
-      "devOptional": true,
+      "dev": true,
       "dependencies": {
         "tslib": "^2.4.0"
       }
@@ -4598,13 +4590,6 @@
         "@types/trusted-types": "*"
       }
     },
-    "node_modules/@types/estree": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
-      "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/@types/json-schema": {
       "version": "7.0.15",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@@ -4636,16 +4621,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/@types/node": {
-      "version": "22.5.5",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.5.tgz",
-      "integrity": "sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "undici-types": "~6.19.2"
-      }
-    },
     "node_modules/@types/parse-json": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz",
@@ -4687,208 +4662,11 @@
       "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
       "dev": true
     },
-    "node_modules/@webassemblyjs/ast": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz",
-      "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@webassemblyjs/helper-numbers": "1.11.6",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6"
-      }
-    },
-    "node_modules/@webassemblyjs/floating-point-hex-parser": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz",
-      "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==",
-      "dev": true,
-      "peer": true
-    },
-    "node_modules/@webassemblyjs/helper-api-error": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz",
-      "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==",
-      "dev": true,
-      "peer": true
-    },
-    "node_modules/@webassemblyjs/helper-buffer": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz",
-      "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw==",
-      "dev": true,
-      "peer": true
-    },
-    "node_modules/@webassemblyjs/helper-numbers": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz",
-      "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@webassemblyjs/floating-point-hex-parser": "1.11.6",
-        "@webassemblyjs/helper-api-error": "1.11.6",
-        "@xtuc/long": "4.2.2"
-      }
-    },
-    "node_modules/@webassemblyjs/helper-wasm-bytecode": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz",
-      "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==",
-      "dev": true,
-      "peer": true
-    },
-    "node_modules/@webassemblyjs/helper-wasm-section": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz",
-      "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-buffer": "1.12.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
-        "@webassemblyjs/wasm-gen": "1.12.1"
-      }
-    },
-    "node_modules/@webassemblyjs/ieee754": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz",
-      "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@xtuc/ieee754": "^1.2.0"
-      }
-    },
-    "node_modules/@webassemblyjs/leb128": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz",
-      "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@xtuc/long": "4.2.2"
-      }
-    },
-    "node_modules/@webassemblyjs/utf8": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz",
-      "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==",
-      "dev": true,
-      "peer": true
-    },
-    "node_modules/@webassemblyjs/wasm-edit": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz",
-      "integrity": "sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-buffer": "1.12.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
-        "@webassemblyjs/helper-wasm-section": "1.12.1",
-        "@webassemblyjs/wasm-gen": "1.12.1",
-        "@webassemblyjs/wasm-opt": "1.12.1",
-        "@webassemblyjs/wasm-parser": "1.12.1",
-        "@webassemblyjs/wast-printer": "1.12.1"
-      }
-    },
-    "node_modules/@webassemblyjs/wasm-gen": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz",
-      "integrity": "sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
-        "@webassemblyjs/ieee754": "1.11.6",
-        "@webassemblyjs/leb128": "1.11.6",
-        "@webassemblyjs/utf8": "1.11.6"
-      }
-    },
-    "node_modules/@webassemblyjs/wasm-opt": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz",
-      "integrity": "sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-buffer": "1.12.1",
-        "@webassemblyjs/wasm-gen": "1.12.1",
-        "@webassemblyjs/wasm-parser": "1.12.1"
-      }
-    },
-    "node_modules/@webassemblyjs/wasm-parser": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz",
-      "integrity": "sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-api-error": "1.11.6",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
-        "@webassemblyjs/ieee754": "1.11.6",
-        "@webassemblyjs/leb128": "1.11.6",
-        "@webassemblyjs/utf8": "1.11.6"
-      }
-    },
-    "node_modules/@webassemblyjs/wast-printer": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz",
-      "integrity": "sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@xtuc/long": "4.2.2"
-      }
-    },
-    "node_modules/@xtuc/ieee754": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",
-      "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==",
-      "dev": true,
-      "peer": true
-    },
-    "node_modules/@xtuc/long": {
-      "version": "4.2.2",
-      "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz",
-      "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/abortcontroller-polyfill": {
       "version": "1.7.5",
       "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.5.tgz",
-      "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ=="
-    },
-    "node_modules/acorn": {
-      "version": "8.12.1",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
-      "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==",
-      "dev": true,
-      "peer": true,
-      "bin": {
-        "acorn": "bin/acorn"
-      },
-      "engines": {
-        "node": ">=0.4.0"
-      }
-    },
-    "node_modules/acorn-import-attributes": {
-      "version": "1.9.5",
-      "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz",
-      "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==",
-      "dev": true,
-      "peer": true,
-      "peerDependencies": {
-        "acorn": "^8"
-      }
+      "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ==",
+      "dev": true
     },
     "node_modules/ajv": {
       "version": "6.12.6",
@@ -4993,6 +4771,7 @@
       "version": "3.0.9",
       "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.9.tgz",
       "integrity": "sha512-H7JU6iBHTal1gp56aKoaa//YUxEaAOUiydvrV/pILqIHXTtqxSkATOnDA2u+jZ/61sD+L/412+7kzXRtWukhpQ==",
+      "dev": true,
       "dependencies": {
         "safe-buffer": "^5.0.1"
       }
@@ -5053,13 +4832,6 @@
         "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
       }
     },
-    "node_modules/buffer-from": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
-      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/callsites": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -5150,6 +4922,7 @@
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
       "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==",
+      "dev": true,
       "engines": {
         "node": ">=0.8"
       }
@@ -5359,6 +5132,7 @@
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-7.0.0.tgz",
       "integrity": "sha512-M3NhsLbV1i6HuGzBUH8vXrtxOk+tWmzWKDMbAVSUp3Zsjm7ywFeuwrUXhmhQyRK1q5B5GGy7hcXPbj3bnfZg2g==",
+      "dev": true,
       "engines": {
         "node": ">=6"
       }
@@ -5366,7 +5140,8 @@
     "node_modules/dotenv-expand": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz",
-      "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA=="
+      "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==",
+      "dev": true
     },
     "node_modules/electron-to-chromium": {
       "version": "1.4.788",
@@ -5382,20 +5157,6 @@
         "node": ">= 4"
       }
     },
-    "node_modules/enhanced-resolve": {
-      "version": "5.17.1",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz",
-      "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "graceful-fs": "^4.2.4",
-        "tapable": "^2.2.0"
-      },
-      "engines": {
-        "node": ">=10.13.0"
-      }
-    },
     "node_modules/entities": {
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
@@ -5424,13 +5185,6 @@
         "is-arrayish": "^0.2.1"
       }
     },
-    "node_modules/es-module-lexer": {
-      "version": "1.5.4",
-      "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
-      "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/escalade": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz",
@@ -5447,53 +5201,6 @@
         "node": ">=0.8.0"
       }
     },
-    "node_modules/eslint-scope": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
-      "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "esrecurse": "^4.3.0",
-        "estraverse": "^4.1.1"
-      },
-      "engines": {
-        "node": ">=8.0.0"
-      }
-    },
-    "node_modules/esrecurse": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
-      "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "estraverse": "^5.2.0"
-      },
-      "engines": {
-        "node": ">=4.0"
-      }
-    },
-    "node_modules/esrecurse/node_modules/estraverse": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
-      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
-      "dev": true,
-      "peer": true,
-      "engines": {
-        "node": ">=4.0"
-      }
-    },
-    "node_modules/estraverse": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
-      "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
-      "dev": true,
-      "peer": true,
-      "engines": {
-        "node": ">=4.0"
-      }
-    },
     "node_modules/esutils": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
@@ -5503,16 +5210,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/events": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
-      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
-      "dev": true,
-      "peer": true,
-      "engines": {
-        "node": ">=0.8.x"
-      }
-    },
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -5603,13 +5300,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/glob-to-regexp": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
-      "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/globals": {
       "version": "11.12.0",
       "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
@@ -5618,13 +5308,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/graceful-fs": {
-      "version": "4.2.11",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
-      "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -5830,47 +5513,6 @@
         "node": ">=0.12.0"
       }
     },
-    "node_modules/jest-worker": {
-      "version": "27.5.1",
-      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz",
-      "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@types/node": "*",
-        "merge-stream": "^2.0.0",
-        "supports-color": "^8.0.0"
-      },
-      "engines": {
-        "node": ">= 10.13.0"
-      }
-    },
-    "node_modules/jest-worker/node_modules/has-flag": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
-      "dev": true,
-      "peer": true,
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/jest-worker/node_modules/supports-color": {
-      "version": "8.1.1",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
-      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "has-flag": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/supports-color?sponsor=1"
-      }
-    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -6171,16 +5813,6 @@
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
       "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="
     },
-    "node_modules/loader-runner": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
-      "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==",
-      "dev": true,
-      "peer": true,
-      "engines": {
-        "node": ">=6.11.5"
-      }
-    },
     "node_modules/loader-utils": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz",
@@ -6270,13 +5902,6 @@
       "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==",
       "license": "MIT"
     },
-    "node_modules/merge-stream": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
-      "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/micromatch": {
       "version": "4.0.8",
       "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
@@ -6289,29 +5914,6 @@
         "node": ">=8.6"
       }
     },
-    "node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true,
-      "peer": true,
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "mime-db": "1.52.0"
-      },
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
     "node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -6374,13 +5976,6 @@
         "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
       }
     },
-    "node_modules/neo-async": {
-      "version": "2.6.2",
-      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
-      "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/node-addon-api": {
       "version": "7.1.0",
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.0.tgz",
@@ -9155,16 +8750,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/randombytes": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
-      "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "safe-buffer": "^5.1.0"
-      }
-    },
     "node_modules/react": {
       "version": "18.3.1",
       "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
@@ -9303,6 +8888,7 @@
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
       "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+      "dev": true,
       "funding": [
         {
           "type": "github",
@@ -9352,16 +8938,6 @@
         "semver": "bin/semver.js"
       }
     },
-    "node_modules/serialize-javascript": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz",
-      "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "randombytes": "^2.1.0"
-      }
-    },
     "node_modules/shallowequal": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz",
@@ -9383,17 +8959,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/source-map-support": {
-      "version": "0.5.21",
-      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz",
-      "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "buffer-from": "^1.0.0",
-        "source-map": "^0.6.0"
-      }
-    },
     "node_modules/stable": {
       "version": "0.1.8",
       "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz",
@@ -9460,16 +9025,6 @@
       "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz",
       "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ=="
     },
-    "node_modules/tapable": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
-      "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==",
-      "dev": true,
-      "peer": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/term-size": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz",
@@ -9482,86 +9037,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/terser": {
-      "version": "5.33.0",
-      "resolved": "https://registry.npmjs.org/terser/-/terser-5.33.0.tgz",
-      "integrity": "sha512-JuPVaB7s1gdFKPKTelwUyRq5Sid2A3Gko2S0PncwdBq7kN9Ti9HPWDQ06MPsEDGsZeVESjKEnyGy68quBk1w6g==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@jridgewell/source-map": "^0.3.3",
-        "acorn": "^8.8.2",
-        "commander": "^2.20.0",
-        "source-map-support": "~0.5.20"
-      },
-      "bin": {
-        "terser": "bin/terser"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/terser-webpack-plugin": {
-      "version": "5.3.10",
-      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz",
-      "integrity": "sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@jridgewell/trace-mapping": "^0.3.20",
-        "jest-worker": "^27.4.5",
-        "schema-utils": "^3.1.1",
-        "serialize-javascript": "^6.0.1",
-        "terser": "^5.26.0"
-      },
-      "engines": {
-        "node": ">= 10.13.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/webpack"
-      },
-      "peerDependencies": {
-        "webpack": "^5.1.0"
-      },
-      "peerDependenciesMeta": {
-        "@swc/core": {
-          "optional": true
-        },
-        "esbuild": {
-          "optional": true
-        },
-        "uglify-js": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/terser-webpack-plugin/node_modules/schema-utils": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz",
-      "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@types/json-schema": "^7.0.8",
-        "ajv": "^6.12.5",
-        "ajv-keywords": "^3.5.2"
-      },
-      "engines": {
-        "node": ">= 10.13.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/webpack"
-      }
-    },
-    "node_modules/terser/node_modules/commander": {
-      "version": "2.20.3",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
-      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/timsort": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/timsort/-/timsort-0.3.0.tgz",
@@ -9608,6 +9083,7 @@
       "version": "5.4.5",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz",
       "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==",
+      "dev": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -9622,13 +9098,6 @@
       "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
       "license": "MIT"
     },
-    "node_modules/undici-types": {
-      "version": "6.19.8",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
-      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
-      "dev": true,
-      "peer": true
-    },
     "node_modules/unicode-canonical-property-names-ecmascript": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz",
@@ -9715,101 +9184,11 @@
         "node": ">= 4"
       }
     },
-    "node_modules/watchpack": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz",
-      "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "glob-to-regexp": "^0.4.1",
-        "graceful-fs": "^4.1.2"
-      },
-      "engines": {
-        "node": ">=10.13.0"
-      }
-    },
     "node_modules/weak-lru-cache": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz",
       "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw=="
     },
-    "node_modules/webpack": {
-      "version": "5.94.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.94.0.tgz",
-      "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@types/estree": "^1.0.5",
-        "@webassemblyjs/ast": "^1.12.1",
-        "@webassemblyjs/wasm-edit": "^1.12.1",
-        "@webassemblyjs/wasm-parser": "^1.12.1",
-        "acorn": "^8.7.1",
-        "acorn-import-attributes": "^1.9.5",
-        "browserslist": "^4.21.10",
-        "chrome-trace-event": "^1.0.2",
-        "enhanced-resolve": "^5.17.1",
-        "es-module-lexer": "^1.2.1",
-        "eslint-scope": "5.1.1",
-        "events": "^3.2.0",
-        "glob-to-regexp": "^0.4.1",
-        "graceful-fs": "^4.2.11",
-        "json-parse-even-better-errors": "^2.3.1",
-        "loader-runner": "^4.2.0",
-        "mime-types": "^2.1.27",
-        "neo-async": "^2.6.2",
-        "schema-utils": "^3.2.0",
-        "tapable": "^2.1.1",
-        "terser-webpack-plugin": "^5.3.10",
-        "watchpack": "^2.4.1",
-        "webpack-sources": "^3.2.3"
-      },
-      "bin": {
-        "webpack": "bin/webpack.js"
-      },
-      "engines": {
-        "node": ">=10.13.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/webpack"
-      },
-      "peerDependenciesMeta": {
-        "webpack-cli": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/webpack-sources": {
-      "version": "3.2.3",
-      "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz",
-      "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==",
-      "dev": true,
-      "peer": true,
-      "engines": {
-        "node": ">=10.13.0"
-      }
-    },
-    "node_modules/webpack/node_modules/schema-utils": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz",
-      "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==",
-      "dev": true,
-      "peer": true,
-      "dependencies": {
-        "@types/json-schema": "^7.0.8",
-        "ajv": "^6.12.5",
-        "ajv-keywords": "^3.5.2"
-      },
-      "engines": {
-        "node": ">= 10.13.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/webpack"
-      }
-    },
     "node_modules/yallist": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",

extensions/react-widget/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docsgpt",
-  "version": "0.4.2",
+  "version": "0.4.7",
   "private": false,
   "description": "DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.",
   "source": "./src/index.html",

extensions/react-widget/src/components/DocsGPTWidget.tsx

@@ -9,6 +9,7 @@ import { ThemeProvider } from 'styled-components';
 import Like from "../assets/like.svg"
 import Dislike from "../assets/dislike.svg"
 import MarkdownIt from 'markdown-it';
+
 const themes = {
   dark: {
     bg: '#222327',
@@ -35,41 +36,20 @@ const themes = {
     }
   }
 }
-const GlobalStyles = createGlobalStyle`
-.response pre {
-    padding: 8px;
-    width: 90%;
-    font-size: 12px;
-    border-radius: 6px;
-    overflow-x: auto;
-    background-color: #1B1C1F;
-    color: #fff !important;
-}
-.response h1{
-  font-size: 20px;
-}
-.response h2{
-  font-size: 18px;
-}
-.response h3{
-  font-size: 16px;
-}
-.response p{
-  margin:0px;
-}
-.response code:not(pre code){
-  border-radius: 6px;
-  padding: 1px 3px 1px 3px;
-  font-size: 12px;
-  display: inline-block;
-  background-color: #646464;
-  color: #fff !important;
-}
-.response code {
-  white-space: pre-wrap !important;
-  line-break: loose !important;
-}
-`;
+
+const sizesConfig = {
+  small: { size: 'small', width: '320px', height: '400px' },
+  medium: { size: 'medium', width: '400px', height: '80vh' },
+  large: { size: 'large', width: '666px', height: '75vh' },
+  getCustom: (custom: { width: string; height: string; maxWidth?: string; maxHeight?: string }) => ({
+    size: 'custom',
+    width: custom.width,
+    height: custom.height,
+    maxWidth: custom.maxWidth || '968px',
+    maxHeight: custom.maxHeight || '70vh',
+  }),
+};
+
 const Overlay = styled.div`
   position: fixed;
   top: 0;
@@ -80,54 +60,160 @@ const Overlay = styled.div`
   z-index: 999;
   transition: opacity 0.5s;
 `
-const WidgetContainer = styled.div<{ modal: boolean }>`
-    display: block;
+const WidgetContainer = styled.div<{ modal?: boolean, isOpen?: boolean }>`
+    all: initial;
     position: fixed;
     right: ${props => props.modal ? '50%' : '10px'};
     bottom: ${props => props.modal ? '50%' : '10px'};
     z-index: 1000;
-    display: flex;
+    display: none;
+    transform-origin:100% 100%;
+    &.open {
+        animation: createBox 250ms cubic-bezier(0.25, 0.1, 0.25, 1) forwards;
+    }
+    &.close {
+      animation: closeBox 250ms cubic-bezier(0.25, 0.1, 0.25, 1) forwards;
+    }
     ${props => props.modal &&
     "transform : translate(50%,50%);"
   }
-    flex-direction: column;
     align-items: center;
     text-align: left;
-    @media only screen and (max-width: 768px) {
-    max-height: 100vh !important;
-    overflow: auto;
+    @keyframes createBox {
+      0% {
+        transform: scale(0.6);
+      }
+      90% {
+        transform: scale(1.02);
+      }
+      100% {
+        transform: scale(1);
+      }
+    }
+
+    @keyframes closeBox {
+      0% {
+        transform: scale(1); 
+      }
+      10% {
+        transform: scale(1.02); 
+      }
+      100% {
+        transform: scale(0.6);
+      }
     }
 `;
-const StyledContainer = styled.div`
-    display: flex;
+const StyledContainer = styled.div<{ isOpen: boolean }>`
+    all: initial;
+    max-height: ${(props) => props.theme.dimensions.maxHeight};
+    max-width: ${(props) => props.theme.dimensions.maxWidth};
     position: relative;
     flex-direction: column;
-    justify-content: center;
+    justify-content: space-between;
     bottom: 0;
     left: 0;
-    border-radius: 0.75rem;
-    background-color: ${props => props.theme.primary.bg};
+    background-color: ${(props) => props.theme.primary.bg};
     font-family: sans-serif;
-    box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05), 0 2px 4px rgba(0, 0, 0, 0.1);
-    transition: visibility 0.3s, opacity 0.3s;
-`;
-const FloatingButton = styled.div<{ bgcolor: string }>`
-    position: fixed;
     display: flex;
+    border-radius: 12px;
+    box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05), 0 2px 4px rgba(0, 0, 0, 0.1);
+    padding: 26px 26px 0px 26px;
+    animation: ${({ isOpen, theme }) =>
+      theme.dimensions.size === 'large'
+        ? isOpen
+          ? 'fadeIn 150ms ease-in forwards'
+          : 'fadeOut 150ms ease-in forwards'
+        : isOpen
+        ? 'openContainer 150ms ease-in forwards'
+        : 'closeContainer 250ms ease-in forwards'};
+    @keyframes openContainer {
+      0% {
+        width: 200px;
+        height: 100px;
+      }
+      100% {
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+        border-radius: 12px;
+      }
+    }
+    @keyframes closeContainer {
+      0% {
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+        border-radius: 12px;
+      }
+      100% {
+        width: 200px;
+        height: 100px;
+      }
+    }
+    @keyframes fadeIn {
+      from {
+        opacity: 0;
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+        transform: scale(0.9);
+      }
+      to {
+        opacity: 1;
+        transform: scale(1);
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+      }
+    }
+    @keyframes fadeOut {
+      from {
+        opacity: 1;
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+      }
+      to {
+        opacity: 0;
+        transform: scale(0.9);
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+      }
+    }
+    @media only screen and (max-width: 768px) {
+      max-height: 100vh;
+      max-width: 80vw;
+      overflow: auto;
+    }
+`;
+const FloatingButton = styled.div<{ bgcolor: string, hidden: boolean, isAnimatingButton: boolean }>`
+    position: fixed;
+    display: ${props => props.hidden ? "none" : "flex"};
     z-index: 500;
     justify-content: center;
+    gap: 8px;
+    padding: 14px;
     align-items: center;
-    bottom: 1rem;
-    right: 1rem;
-    width: 5rem;
-    height: 5rem;
+    bottom: 16px;
+    color: white;
+    font-family: sans-serif;
+    right: 16px;
+    font-weight: 500;
     border-radius: 9999px;
     background: ${props => props.bgcolor};
     box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
     cursor: pointer;
+    animation: ${props => props.isAnimatingButton ? 'scaleAnimation 200ms forwards' : 'none'};
     &:hover {
-        transform: scale(1.1);
-        transition: transform 0.2s ease-in-out;
+      transform: scale(1.1);
+      transition: transform 0.2s ease-in-out;
+    }
+    &:not(:hover) {
+      transition: transform 0.2s ease-in-out;
+    }
+
+    @keyframes scaleAnimation {
+      from {
+      transform: scale(1.2);
+      }
+      to {
+      transform: scale(1);
+      }
     }
 `;
 const CancelButton = styled.button`
@@ -135,7 +221,7 @@ const CancelButton = styled.button`
     position: absolute;
     top: 0;
     right: 0;
-    margin: 0.5rem;
+    margin: 8px;
     width: 30px;
     padding: 0;
     background-color: transparent;
@@ -154,52 +240,37 @@ const CancelButton = styled.button`
 
 const Header = styled.div`
     display: flex;
-    align-items: center;
-    padding-inline: 0.75rem;
-    padding-top: 1rem;
-    padding-bottom: 0.5rem;
-`;
-
-const IconWrapper = styled.div`
-    padding: 0.5rem;
+    align-items: flex-start;
 `;
 
 const ContentWrapper = styled.div`
-    flex: 1;
-    margin-left: 0.5rem;
+    display: flex;
+    flex-direction: column;
+    gap:2px; 
+    margin-left: 8px;
 `;
 
 const Title = styled.h3`
-    font-size: 1rem;
+    font-size: 14px;
     font-weight: normal;
     color: ${props => props.theme.primary.text};
-    margin-top: 0;
-    margin-bottom: 0.25rem;
+    margin: 0;
 `;
 
 const Description = styled.p`
-    font-size: 0.85rem;
+    font-size: 13.75px;
     color: ${props => props.theme.secondary.text};
-    margin-top: 0;
+    margin: 0 ;
+    padding: 0 ;
 `;
 
-const Conversation = styled.div<{ size: string }>`
-    min-height: 250px;
-    max-width: 968px;
-    height: ${props => props.size === 'large' ? '75vh' : props.size === 'medium' ? '70vh' : '320px'};
-    width: ${props => props.size === 'large' ? '60vw' : props.size === 'medium' ? '28vw' : '400px'};
-    padding-inline: 0.5rem;
-    border-radius: 0.375rem;
-    text-align: left;
-    overflow-y: auto;
-    scrollbar-width: thin;
-    scrollbar-color: #4a4a4a transparent; /* thumb color track color */
-    @media only screen and (max-width: 768px) {
-    width: 90vw !important;
-    }
-    @media only screen and (min-width:768px ) and (max-width: 1280px) {
-    width:${props => props.size === 'large' ? '90vw' : props.size === 'medium' ? '60vw' : '400px'} !important;
-    }
+const Conversation = styled.div`
+  height: 70%;
+  border-radius: 6px;
+  text-align: left;
+  overflow-y: auto;
+  scrollbar-width: thin;
+  scrollbar-color: #4a4a4a transparent; /* thumb color track color */
 `;
 const Feedback = styled.div`
   background-color: transparent;
@@ -215,9 +286,9 @@ const MessageBubble = styled.div<{ type: MESSAGE_TYPE }>`
     position: relative;
     width: 100%;;
     float: right;
-    margin: 0rem;
+    margin: 0px;
     &:hover ${Feedback} * {
-    visibility: visible !important;
+    visibility: visible ;
   }
 `;
 const Message = styled.div<{ type: MESSAGE_TYPE }>`
@@ -232,19 +303,66 @@ const Message = styled.div<{ type: MESSAGE_TYPE }>`
     margin: 4px;
     display: block;
     line-height: 1.5;
-    padding: 0.75rem;
-    border-radius: 0.375rem;
+    padding: 12px;
+    border-radius: 6px;
 `;
+const Markdown = styled.div`
+ pre {
+      padding: 8px;
+      width: 90%;
+      font-size: 12px;
+      border-radius: 6px;
+      overflow-x: auto;
+      background-color: #1B1C1F;
+      color: #fff ;
+    }
+
+    h1 {
+      font-size: 16px;
+    }
+
+    h2 {
+      font-size: 14px;
+    }
+
+    h3 {
+      font-size: 14px;
+    }
+
+    p {
+      margin: 0px;
+    }
+
+    code:not(pre code) {
+      border-radius: 6px;
+      padding: 1px 3px;
+      font-size: 12px;
+      display: inline-block;
+      background-color: #646464;
+      color: #fff ;
+    }
+
+    code {
+      white-space: pre-wrap ;
+      overflow-wrap: break-word;
+      word-break: break-all;
+    }
+
+    ul{
+      padding:0px;
+      list-style-position: inside;
+    }
+`
 const ErrorAlert = styled.div`
   color: #b91c1c;
   border:0.1px solid #b91c1c;
   display: flex;
   padding:4px;
-  margin:0.7rem;
+  margin:11.2px;
   opacity: 90%;
   max-width: 70%;
   font-weight: 400;
-  border-radius: 0.375rem;
+  border-radius: 6px;
   justify-content: space-evenly;
 `
 //dot loading animation
@@ -265,10 +383,9 @@ const DotAnimation = styled.div`
 const Delay = styled(DotAnimation) <{ delay: number }>`
   animation-delay: ${props => props.delay + 'ms'};
 `;
-const PromptContainer = styled.form<{ size: string }>`
+const PromptContainer = styled.form`
   background-color: transparent;
-  height: ${props => props.size == 'large' ? '60px' : '40px'};
-  margin: 16px;
+  height: ${props => props.theme.dimensions.size == 'large' ? '60px' : '40px'};
   display: flex;
   justify-content: space-evenly;
 `;
@@ -282,16 +399,18 @@ const StyledInput = styled.input`
   color: ${props => props.theme.text};
   outline: none;
 `;
-const StyledButton = styled.button<{ size: string }>`
+const StyledButton = styled.button`
   display: flex;
   justify-content: center;
   align-items: center;
   background-image: linear-gradient(to bottom right, #5AF0EC, #E80D9D);
+  background-color: rgba(0, 0, 0, 0.3);
   border-radius: 6px;
-  min-width: ${props => props.size === 'large' ? '60px' : '36px'};
-  height: ${props => props.size === 'large' ? '60px' : '36px'};
+  min-width: ${props => props.theme.dimensions.size === 'large' ? '60px' : '40px'};
+  height: ${props => props.theme.dimensions.size === 'large' ? '60px' : '40px'};
   margin-left:8px;
   padding: 0px;
+  
   border: none;
   cursor: pointer;
   outline: none;
@@ -299,59 +418,64 @@ const StyledButton = styled.button<{ size: string }>`
     opacity: 90%;
   }
   &:disabled {
-    opacity: 60%;
+    background-image: linear-gradient(to bottom right, #2d938f, #b31877);
   }`;
 const HeroContainer = styled.div`
-  position: absolute;
-  top: 50%;
-  left: 50%;
-  display: flex;
-  justify-content: center;
-  align-items: middle;
-  transform: translate(-50%, -50%);
-  width: 80%;
+  position: relative;
+  width: 90%;
   max-width: 500px;
   background-image: linear-gradient(to bottom right, #5AF0EC, #ff1bf4);
   border-radius: 10px;
-  margin: 0 auto;
+  margin: 16px auto;
   padding: 2px;
 `;
 const HeroWrapper = styled.div`
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+  gap: 8px;
+  align-items: middle;
   background-color: ${props => props.theme.primary.bg};
   border-radius: 10px; 
   font-weight: normal;
-  padding: 6px;
-  display: flex;
-  justify-content: space-between;
+  padding: 12px;
 `
 const HeroTitle = styled.h3`
   color: ${props => props.theme.text};
-  margin-bottom: 5px;
-  padding: 2px;
+  font-size: 16px;
+  margin:0px ;
+  padding: 0px;
 `;
 const HeroDescription = styled.p`
   color: ${props => props.theme.text};
-  font-size: 14px;
+  font-size: 12px;
   line-height: 1.5;
+  margin: 0px;
+  padding: 0px;
 `;
+const Hyperlink = styled.a`
+  color: #9971EC;
+  text-decoration: none;
+`;
+const Tagline = styled.div`
+  text-align: center;
+  display: block;
+  color: ${props => props.theme.secondary.text};
+  padding: 12px ;
+  font-size: 12px;
+`;
+
+
 
 const Hero = ({ title, description, theme }: { title: string, description: string, theme: string }) => {
   return (
-    <>
-      <HeroContainer>
-        <HeroWrapper>
-          <IconWrapper style={{ marginTop: '12px' }}>
-            <RocketIcon color={theme === 'light' ? 'black' : 'white'} width={20} height={20} />
-          </IconWrapper>
-          <div>
-            <HeroTitle>{title}</HeroTitle>
-            <HeroDescription>
-              {description}
-            </HeroDescription>
-          </div>
-        </HeroWrapper>
-      </HeroContainer>
-    </>
+    <HeroContainer>
+      <HeroWrapper>
+        <RocketIcon color={theme === 'light' ? 'black' : 'white'} width={24} height={24} />
+        <HeroTitle>{title}</HeroTitle>
+        <HeroDescription>{description}</HeroDescription>
+      </HeroWrapper>
+    </HeroContainer>
   );
 };
 export const DocsGPTWidget = ({
@@ -364,17 +488,22 @@ export const DocsGPTWidget = ({
   heroDescription = 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.',
   size = 'small',
   theme = 'dark',
-  buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/message.svg',
+  buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/chat.svg',
+  buttonText = 'Ask a question',
   buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)',
-  collectFeedback = true
+  collectFeedback = true,
+  deafultOpen = false
 }: WidgetProps) => {
   const [prompt, setPrompt] = React.useState('');
   const [status, setStatus] = React.useState<Status>('idle');
   const [queries, setQueries] = React.useState<Query[]>([])
   const [conversationId, setConversationId] = React.useState<string | null>(null)
-  const [open, setOpen] = React.useState<boolean>(false)
+  const [open, setOpen] = React.useState<boolean>(deafultOpen)
   const [eventInterrupt, setEventInterrupt] = React.useState<boolean>(false); //click or scroll by user while autoScrolling
+  const [isAnimatingButton, setIsAnimatingButton] = React.useState(false);
+  const [isFloatingButtonVisible, setIsFloatingButtonVisible] = React.useState(true);
   const isBubbleHovered = useRef<boolean>(false)
+  const widgetRef = useRef<HTMLDivElement>(null)
   const endMessageRef = React.useRef<HTMLDivElement | null>(null);
   const md = new MarkdownIt();
 
@@ -417,7 +546,7 @@ export const DocsGPTWidget = ({
             });
           }
         })
-        .catch(err => console.log("Connection failed",err))
+        .catch(err => console.log("Connection failed", err))
     }
     else {
       delete query.feedback;
@@ -453,8 +582,11 @@ export const DocsGPTWidget = ({
               setQueries(updatedQueries);
               setStatus('idle')
             }
+            else if (data.type === 'source') {
+              // handle the case where data type === 'source'
+            }
             else {
-              const result = data.answer;
+              const result = data.answer ? data.answer : ''; //Fallback to an empty string if data.answer is undefined
               const streamingResponse = queries[queries.length - 1].response ? queries[queries.length - 1].response : '';
               const updatedQueries = [...queries];
               updatedQueries[updatedQueries.length - 1].response = streamingResponse + result;
@@ -483,34 +615,50 @@ export const DocsGPTWidget = ({
   const handleImageError = (event: React.SyntheticEvent<HTMLImageElement, Event>) => {
     event.currentTarget.src = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png";
   };
+  const handleClose = () => {
+    setOpen(false);
+    setTimeout(() => {
+      if (widgetRef.current) widgetRef.current.style.display = "none";
+      setIsFloatingButtonVisible(true);
+      setIsAnimatingButton(true);
+      setTimeout(() => setIsAnimatingButton(false), 200);
+    }, 250)
+  };
+  const handleOpen = () => {
+    setOpen(true);
+    setIsFloatingButtonVisible(false);
+    if (widgetRef.current)
+      widgetRef.current.style.display = 'block'
+  }
+  const dimensions =
+    typeof size === 'object' && 'custom' in size
+      ? sizesConfig.getCustom(size.custom)
+      : sizesConfig[size];
+
   return (
-    <ThemeProvider theme={themes[theme]}>
+    <ThemeProvider theme={{ ...themes[theme], dimensions }}>
       {open && size === 'large' &&
-        <Overlay onClick={() => {
-          setOpen(false)
-        }} />
+        <Overlay onClick={handleClose} />
       }
-      <FloatingButton bgcolor={buttonBg} onClick={() => setOpen(!open)} hidden={open}>
-        <img style={{ maxHeight: '4rem', maxWidth: '4rem' }} src={buttonIcon} />
+      <FloatingButton bgcolor={buttonBg} onClick={handleOpen} hidden={!isFloatingButtonVisible} isAnimatingButton={isAnimatingButton}>
+        <img width={24} src={buttonIcon} />
+        <span>{buttonText}</span>
       </FloatingButton>
-      <WidgetContainer modal={size == 'large'}>
-        <GlobalStyles />
-        {open && <StyledContainer>
+      <WidgetContainer ref={widgetRef} className={`${size != "large" && (open ? "open" : "close")}`} modal={size == 'large'}>
+        {<StyledContainer isOpen={open}>
           <div>
-            <CancelButton onClick={() => setOpen(false)}>
+            <CancelButton onClick={handleClose}>
               <Cross2Icon width={24} height={24} color={theme === 'light' ? 'black' : 'white'} />
             </CancelButton>
             <Header>
-              <IconWrapper>
-                <img style={{ maxWidth: "42px", maxHeight: "42px" }} onError={handleImageError} src={avatar} alt='docs-gpt' />
-              </IconWrapper>
+              <img style={{ transform: 'translateY(-5px)', maxWidth: "42px", maxHeight: "42px" }} onError={handleImageError} src={avatar} alt='docs-gpt' />
               <ContentWrapper>
                 <Title>{title}</Title>
                 <Description>{description}</Description>
               </ContentWrapper>
             </Header>
           </div>
-          <Conversation size={size} onWheel={handleUserInterrupt} onTouchMove={handleUserInterrupt}>
+          <Conversation onWheel={handleUserInterrupt} onTouchMove={handleUserInterrupt}>
             {
               queries.length > 0 ? queries?.map((query, index) => {
                 return (
@@ -530,8 +678,7 @@ export const DocsGPTWidget = ({
                           type='ANSWER'
                           ref={(index === queries.length - 1) ? endMessageRef : null}
                         >
-                          <div
-                            className="response"
+                          <Markdown
                             dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(md.render(query.response)) }}
                           />
                         </Message>
@@ -557,9 +704,8 @@ export const DocsGPTWidget = ({
                         : <div>
                           {
                             query.error ? <ErrorAlert>
-                              <IconWrapper>
-                                <ExclamationTriangleIcon style={{ marginTop: '4px' }} width={22} height={22} color='#b91c1c' />
-                              </IconWrapper>
+
+                              <ExclamationTriangleIcon width={22} height={22} color='#b91c1c' />
                               <div>
                                 <h5 style={{ margin: 2 }}>Network Error</h5>
                                 <span style={{ margin: 2, fontSize: '13px' }}>{query.error}</span>
@@ -580,18 +726,22 @@ export const DocsGPTWidget = ({
                 : <Hero title={heroTitle} description={heroDescription} theme={theme} />
             }
           </Conversation>
-          <PromptContainer
-            size={size}
-            onSubmit={handleSubmit}>
-            <StyledInput
-              value={prompt} onChange={(event) => setPrompt(event.target.value)}
-              type='text' placeholder="What do you want to do?" />
-            <StyledButton
-              size={size}
-              disabled={prompt.trim().length == 0 || status !== 'idle'}>
-              <PaperPlaneIcon width={15} height={15} color='white' />
-            </StyledButton>
-          </PromptContainer>
+          <div>
+            <PromptContainer
+              onSubmit={handleSubmit}>
+              <StyledInput
+                value={prompt} onChange={(event) => setPrompt(event.target.value)}
+                type='text' placeholder="Ask your question" />
+              <StyledButton
+                disabled={prompt.trim().length == 0 || status !== 'idle'}>
+                <PaperPlaneIcon width={18} height={18} color='white' />
+              </StyledButton>
+            </PromptContainer>
+            <Tagline>
+              Powered by&nbsp;
+              <Hyperlink target='_blank' href='https://www.docsgpt.cloud/'>DocsGPT</Hyperlink>
+            </Tagline>
+          </div>
         </StyledContainer>}
       </WidgetContainer>
     </ThemeProvider>

extensions/react-widget/src/types/index.ts

@@ -19,9 +19,18 @@ export interface WidgetProps {
   description?: string;
   heroTitle?: string;
   heroDescription?: string;
-  size?: 'small' | 'medium' | 'large';
+  size?: 'small' | 'medium' | 'large' | {
+    custom: {
+      width: string;
+      height: string;
+      maxWidth?: string;
+      maxHeight?: string;
+    };
+  };
   theme?:THEME,
   buttonIcon?:string;
+  buttonText?:string;
   buttonBg?:string;
-  collectFeedback?:boolean
+  collectFeedback?:boolean;
+  deafultOpen?: boolean;
 }

extensions/slack-bot/.gitignore

@@ -0,0 +1,3 @@
+.env
+.venv/
+get-pip.py

extensions/slack-bot/Readme.md

@@ -0,0 +1,84 @@
+
+# Slack Bot Configuration Guide
+
+> **Note:** The following guidelines must be followed on the [Slack API website](https://api.slack.com/) for setting up your Slack app and generating the necessary tokens.
+
+## Step-by-Step Instructions
+
+### 1. Navigate to Your Apps
+- Go to the Slack API page for apps and select **Create an App** from the “From Scratch” option.
+
+### 2. App Creation
+- Name your app and choose the workspace where you wish to add the assistant. 
+
+### 3. Enabling Socket Mode
+- Navigate to **Settings > Socket Mode** and enable **Socket Mode**. 
+- This action will generate an App-level token. Select the `connections:write` scope and copy the App-level token for future use.
+
+### 4. Socket Naming
+- Assign a name to your socket as per your preference.
+
+### 5. Basic Information Setup
+- Go to **Basic Information** (under **Settings**) and configure the following:
+  - Assistant name
+  - App icon
+  - Background color 
+
+### 6. Bot Token and Permissions
+- In the **OAuth & Permissions** option found under the **Features** section, retrieve the Bot Token. Save it for future usage.
+- You will also need to add specific bot token scopes:
+  - `app_mentions:read`
+  - `assistant:write`
+  - `chat:write`
+  - `chat:write.public`
+  - `im:history`
+
+### 7. Enable Events
+- From **Event Subscriptions**, enable events and add the following Bot User events:
+  - `app_mention`
+  - `assistant_thread_context_changed`
+  - `assistant_thread_started`
+  - `message.im`
+
+### 8. Agent/Assistant Toggle
+- In the **Features > Agent & Assistants** section, toggle on the Agent or Assistant option. 
+- In the **Suggested Prompts** setting, leave it as `dynamic` (this is the default setting).
+
+---
+
+## Code-Side Configuration Guide
+
+This section focuses on generating and setting up the necessary tokens in the `.env` file, using the `.env-example` as a template.
+
+### Step 1: Generating Required Keys
+
+1. **SLACK_APP_TOKEN**
+   - Navigate to **Settings > Socket Mode** in the Slack API and enable **Socket Mode**.
+   - Copy the App-level token generated (usually starts with `xapp-`).
+
+2. **SLACK_BOT_TOKEN**
+   - Go to **OAuth & Permissions** (under the **Features** section in Slack API).
+   - Retrieve the **Bot Token** (starts with `xoxb-`).
+
+3. **DOCSGPT_API_KEY**
+   - Go to the **DocsGPT website**.
+   - Navigate to **Settings > Chatbots > Create New** to generate a DocsGPT API Key.
+   - Copy the generated key for use.
+
+### Step 2: Creating and Updating the `.env` File
+
+1. Create a new `.env` file in the root of your project (if it doesn’t already exist).
+2. Use the `.env-example` as a reference and update the file with the following keys and values:
+
+```bash
+# .env file
+SLACK_APP_TOKEN=xapp-your-generated-app-token
+SLACK_BOT_TOKEN=xoxb-your-generated-bot-token
+DOCSGPT_API_KEY=your-docsgpt-generated-api-key
+```
+
+Replace the placeholder values with the actual tokens generated from the Slack API and DocsGPT as per the steps outlined above.
+
+---
+
+This concludes the guide for both setting up the Slack API and configuring the `.env` file on the code side.

extensions/slack-bot/app.py

@@ -0,0 +1,112 @@
+import os
+import hashlib
+import httpx
+import re
+from slack_bolt.async_app import AsyncApp
+from slack_bolt.adapter.socket_mode.async_handler import AsyncSocketModeHandler
+from dotenv import load_dotenv
+
+load_dotenv()
+API_BASE = os.getenv("API_BASE", "https://gptcloud.arc53.com")
+API_URL =  API_BASE + "/api/answer"
+
+# Slack bot token and signing secret
+SLACK_BOT_TOKEN = os.getenv("SLACK_BOT_TOKEN")
+SLACK_APP_TOKEN = os.getenv("SLACK_APP_TOKEN")
+
+# OpenAI API key for DocsGPT (replace this with your actual API key)
+DOCSGPT_API_KEY = os.getenv("DOCSGPT_API_KEY")
+
+# Initialize Slack app
+app = AsyncApp(token=SLACK_BOT_TOKEN)
+
+def encode_conversation_id(conversation_id: str) -> str:
+    """
+        Encodes 11 length Slack conversation_id to 12 length string
+        Args:
+        conversation_id (str): The 11 digit slack conversation_id.
+        Returns:
+            str: Hashed id.
+    """    
+    # Create a SHA-256 hash of the string
+    hashed_id = hashlib.sha256(conversation_id.encode()).hexdigest()
+
+    # Take the first 24 characters of the hash
+    hashed_24_char_id = hashed_id[:24]
+    return hashed_24_char_id
+
+async def generate_answer(question: str, messages: list, conversation_id: str | None) -> dict:
+    """Generates an answer using the external API."""
+    payload = {
+        "question": question,
+        "api_key": DOCSGPT_API_KEY,
+        "history": messages,
+        "conversation_id": conversation_id,
+    }
+    headers = {
+        "Content-Type": "application/json; charset=utf-8"
+    }
+    timeout = 60.0
+    async with httpx.AsyncClient() as client:
+        response = await client.post(API_URL, json=payload, headers=headers, timeout=timeout)
+
+        if response.status_code == 200:
+            data = response.json()
+            conversation_id = data.get("conversation_id")
+            answer = data.get("answer", "Sorry, I couldn't find an answer.")
+            return {"answer": answer, "conversation_id": conversation_id}
+        else:
+            print(response.json())
+            return {"answer": "Sorry, I couldn't find an answer.", "conversation_id": None}
+
+@app.message(".*")
+async def message_docs(message, say):
+    client = app.client
+    channel = message['channel']
+    thread_ts = message['thread_ts']
+    user_query = message['text']    
+    await client.assistant_threads_setStatus(
+        channel_id = channel,
+        thread_ts = thread_ts,
+        status = "is generating your answer...",
+    )
+
+    docs_gpt_channel_id = encode_conversation_id(thread_ts)
+    
+    # Get response from DocsGPT
+    response = await generate_answer(user_query,[], docs_gpt_channel_id)
+    answer = convert_to_slack_markdown(response['answer'])
+
+    # Respond in Slack
+    await client.chat_postMessage(text = answer, mrkdwn= True, channel= message['channel'],
+        thread_ts = message['thread_ts'],)
+
+def convert_to_slack_markdown(markdown_text: str):
+    # Convert bold **text** to *text* for Slack
+    slack_text = re.sub(r'\*\*(.*?)\*\*', r'*\1*', markdown_text)  # **text** to *text*
+
+    # Convert italics _text_ to _text_ for Slack
+    slack_text = re.sub(r'_(.*?)_', r'_\1_', slack_text)  # _text_ to _text_
+
+    # Convert inline code `code` to `code` (Slack supports backticks for inline code)
+    slack_text = re.sub(r'`(.*?)`', r'`\1`', slack_text)
+
+    # Convert bullet points with single or no spaces to filled bullets (•)
+    slack_text = re.sub(r'^\s{0,1}[-*]\s+', ' • ', slack_text, flags=re.MULTILINE)
+
+    # Convert bullet points with multiple spaces to hollow bullets (◦)
+    slack_text = re.sub(r'^\s{2,}[-*]\s+', '\t◦ ', slack_text, flags=re.MULTILINE)
+
+    # Convert headers (##) to bold in Slack
+    slack_text = re.sub(r'^\s*#{1,6}\s*(.*?)$', r'*\1*', slack_text, flags=re.MULTILINE)
+
+    return slack_text
+
+async def main():
+    handler = AsyncSocketModeHandler(app, os.environ["SLACK_APP_TOKEN"])
+    await handler.start_async()
+
+# Start the app
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())

extensions/slack-bot/requirements.txt

@@ -0,0 +1,10 @@
+aiohttp>=3,<4
+certifi==2024.7.4
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+idna==3.7
+python-dotenv==1.0.1
+sniffio==1.3.1
+slack-bolt==1.21.0
+bson==0.5.10

frontend/package-lock.json

@@ -9,7 +9,6 @@
       "version": "0.0.0",
       "dependencies": {
         "@reduxjs/toolkit": "^2.2.7",
-        "@vercel/analytics": "^0.1.10",
         "chart.js": "^4.4.4",
         "i18next": "^23.15.1",
         "i18next-browser-languagedetector": "^8.0.0",
@@ -24,7 +23,9 @@
         "react-redux": "^8.0.5",
         "react-router-dom": "^6.8.1",
         "react-syntax-highlighter": "^15.5.0",
-        "remark-gfm": "^4.0.0"
+        "rehype-katex": "^7.0.1",
+        "remark-gfm": "^4.0.0",
+        "remark-math": "^6.0.0"
       },
       "devDependencies": {
         "@types/react": "^18.0.27",
@@ -42,14 +43,14 @@
         "eslint-plugin-prettier": "^5.2.1",
         "eslint-plugin-promise": "^6.6.0",
         "eslint-plugin-react": "^7.35.0",
-        "eslint-plugin-unused-imports": "^2.0.0",
+        "eslint-plugin-unused-imports": "^4.1.4",
         "husky": "^8.0.0",
         "lint-staged": "^15.2.10",
         "postcss": "^8.4.41",
         "prettier": "^3.3.3",
-        "prettier-plugin-tailwindcss": "^0.6.6",
+        "prettier-plugin-tailwindcss": "^0.6.8",
         "tailwindcss": "^3.4.11",
-        "typescript": "^4.9.5",
+        "typescript": "^5.6.2",
         "vite": "^5.4.6",
         "vite-plugin-svgr": "^4.2.0"
       }
@@ -1636,6 +1637,12 @@
       "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==",
       "dev": true
     },
+    "node_modules/@types/katex": {
+      "version": "0.16.7",
+      "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.7.tgz",
+      "integrity": "sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==",
+      "license": "MIT"
+    },
     "node_modules/@types/mdast": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz",
@@ -2081,14 +2088,6 @@
       "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
       "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ=="
     },
-    "node_modules/@vercel/analytics": {
-      "version": "0.1.10",
-      "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-0.1.10.tgz",
-      "integrity": "sha512-jjJ8GzcPnQp0cMxpfYoUycMRBtDiaIeyVjZPiEPe99Dj1PdjMzAFYEASiV/hpNsXHkpcNYCveDFh6jnmh0YSDQ==",
-      "peerDependencies": {
-        "react": "^16.8||^17||^18"
-      }
-    },
     "node_modules/@vitejs/plugin-react": {
       "version": "4.3.1",
       "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.3.1.tgz",
@@ -3076,6 +3075,24 @@
       "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
       "dev": true
     },
+    "node_modules/easy-speech": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/easy-speech/-/easy-speech-2.4.0.tgz",
+      "integrity": "sha512-wpMv29DEoeP/eyXr4aXpDqd9DvlXl7aQs7BgfKbjGVxqkmQPgNmpbF5YULaTH5bc/5qrteg5MDfCD2Zd0qr4rQ==",
+      "funding": [
+        {
+          "type": "GitHub",
+          "url": "https://github.com/sponsors/jankapunkt"
+        },
+        {
+          "type": "PayPal",
+          "url": "https://paypal.me/kuesterjan"
+        }
+      ],
+      "engines": {
+        "node": ">= 14.x"
+      }
+    },
     "node_modules/electron-to-chromium": {
       "version": "1.5.11",
       "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.11.tgz",
@@ -3092,7 +3109,6 @@
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
       "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
-      "dev": true,
       "engines": {
         "node": ">=0.12"
       },
@@ -3762,19 +3778,13 @@
       }
     },
     "node_modules/eslint-plugin-unused-imports": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-2.0.0.tgz",
-      "integrity": "sha512-3APeS/tQlTrFa167ThtP0Zm0vctjr4M44HMpeg1P4bK6wItarumq0Ma82xorMKdFsWpphQBlRPzw/pxiVELX1A==",
+      "version": "4.1.4",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-4.1.4.tgz",
+      "integrity": "sha512-YptD6IzQjDardkl0POxnnRBhU1OEePMV0nd6siHaRBbd+lyh6NAhFEobiznKU7kTsSsDeSD62Pe7kAM1b7dAZQ==",
       "dev": true,
-      "dependencies": {
-        "eslint-rule-composer": "^0.3.0"
-      },
-      "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
       "peerDependencies": {
-        "@typescript-eslint/eslint-plugin": "^5.0.0",
-        "eslint": "^8.0.0"
+        "@typescript-eslint/eslint-plugin": "^8.0.0-0 || ^7.0.0 || ^6.0.0 || ^5.0.0",
+        "eslint": "^9.0.0 || ^8.0.0"
       },
       "peerDependenciesMeta": {
         "@typescript-eslint/eslint-plugin": {
@@ -3782,15 +3792,6 @@
         }
       }
     },
-    "node_modules/eslint-rule-composer": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/eslint-rule-composer/-/eslint-rule-composer-0.3.0.tgz",
-      "integrity": "sha512-bt+Sh8CtDmn2OajxvNO+BX7Wn4CIWMpTRm3MaiKPCQcnnlm0CS2mhui6QaoeQugs+3Kj2ESKEEGJUdVafwhiCg==",
-      "dev": true,
-      "engines": {
-        "node": ">=4.0.0"
-      }
-    },
     "node_modules/eslint-scope": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
@@ -4585,6 +4586,193 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/hast-util-from-dom": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/hast-util-from-dom/-/hast-util-from-dom-5.0.0.tgz",
+      "integrity": "sha512-d6235voAp/XR3Hh5uy7aGLbM3S4KamdW0WEgOaU1YoewnuYw4HXb5eRtv9g65m/RFGEfUY1Mw4UqCc5Y8L4Stg==",
+      "license": "ISC",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "hastscript": "^8.0.0",
+        "web-namespaces": "^2.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-from-dom/node_modules/@types/hast": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz",
+      "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "*"
+      }
+    },
+    "node_modules/hast-util-from-dom/node_modules/hast-util-parse-selector": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz",
+      "integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-from-dom/node_modules/hastscript": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-8.0.0.tgz",
+      "integrity": "sha512-dMOtzCEd3ABUeSIISmrETiKuyydk1w0pa+gE/uormcTpSYuaNJPbX1NU3JLyscSLjwAQM8bWMhhIlnCqnRvDTw==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "comma-separated-tokens": "^2.0.0",
+        "hast-util-parse-selector": "^4.0.0",
+        "property-information": "^6.0.0",
+        "space-separated-tokens": "^2.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-from-html": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/hast-util-from-html/-/hast-util-from-html-2.0.3.tgz",
+      "integrity": "sha512-CUSRHXyKjzHov8yKsQjGOElXy/3EKpyX56ELnkHH34vDVw1N1XSQ1ZcAvTyAPtGqLTuKP/uxM+aLkSPqF/EtMw==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "devlop": "^1.1.0",
+        "hast-util-from-parse5": "^8.0.0",
+        "parse5": "^7.0.0",
+        "vfile": "^6.0.0",
+        "vfile-message": "^4.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-from-html-isomorphic": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/hast-util-from-html-isomorphic/-/hast-util-from-html-isomorphic-2.0.0.tgz",
+      "integrity": "sha512-zJfpXq44yff2hmE0XmwEOzdWin5xwH+QIhMLOScpX91e/NSGPsAzNCvLQDIEPyO2TXi+lBmU6hjLIhV8MwP2kw==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "hast-util-from-dom": "^5.0.0",
+        "hast-util-from-html": "^2.0.0",
+        "unist-util-remove-position": "^5.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-from-html-isomorphic/node_modules/@types/hast": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz",
+      "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "*"
+      }
+    },
+    "node_modules/hast-util-from-html/node_modules/@types/hast": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz",
+      "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "*"
+      }
+    },
+    "node_modules/hast-util-from-parse5": {
+      "version": "8.0.1",
+      "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.1.tgz",
+      "integrity": "sha512-Er/Iixbc7IEa7r/XLtuG52zoqn/b3Xng/w6aZQ0xGVxzhw5xUFxcRqdPzP6yFi/4HBYRaifaI5fQ1RH8n0ZeOQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "@types/unist": "^3.0.0",
+        "devlop": "^1.0.0",
+        "hastscript": "^8.0.0",
+        "property-information": "^6.0.0",
+        "vfile": "^6.0.0",
+        "vfile-location": "^5.0.0",
+        "web-namespaces": "^2.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-from-parse5/node_modules/@types/hast": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz",
+      "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "*"
+      }
+    },
+    "node_modules/hast-util-from-parse5/node_modules/hast-util-parse-selector": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz",
+      "integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-from-parse5/node_modules/hastscript": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-8.0.0.tgz",
+      "integrity": "sha512-dMOtzCEd3ABUeSIISmrETiKuyydk1w0pa+gE/uormcTpSYuaNJPbX1NU3JLyscSLjwAQM8bWMhhIlnCqnRvDTw==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "comma-separated-tokens": "^2.0.0",
+        "hast-util-parse-selector": "^4.0.0",
+        "property-information": "^6.0.0",
+        "space-separated-tokens": "^2.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-is-element": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz",
+      "integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-is-element/node_modules/@types/hast": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz",
+      "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "*"
+      }
+    },
     "node_modules/hast-util-parse-selector": {
       "version": "2.2.5",
       "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz",
@@ -4628,6 +4816,31 @@
         "@types/unist": "*"
       }
     },
+    "node_modules/hast-util-to-text": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz",
+      "integrity": "sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "@types/unist": "^3.0.0",
+        "hast-util-is-element": "^3.0.0",
+        "unist-util-find-after": "^5.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hast-util-to-text/node_modules/@types/hast": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz",
+      "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "*"
+      }
+    },
     "node_modules/hast-util-whitespace": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz",
@@ -5424,6 +5637,31 @@
         "node": ">=4.0"
       }
     },
+    "node_modules/katex": {
+      "version": "0.16.11",
+      "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.11.tgz",
+      "integrity": "sha512-RQrI8rlHY92OLf3rho/Ts8i/XvjgguEjOkO1BEXcU3N8BqPpSzBNwV/G0Ukr+P/l3ivvJUE/Fa/CwbS6HesGNQ==",
+      "funding": [
+        "https://opencollective.com/katex",
+        "https://github.com/sponsors/katex"
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "commander": "^8.3.0"
+      },
+      "bin": {
+        "katex": "cli.js"
+      }
+    },
+    "node_modules/katex/node_modules/commander": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz",
+      "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 12"
+      }
+    },
     "node_modules/levn": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
@@ -5840,6 +6078,34 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/mdast-util-math": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/mdast-util-math/-/mdast-util-math-3.0.0.tgz",
+      "integrity": "sha512-Tl9GBNeG/AhJnQM221bJR2HPvLOSnLE/T9cJI9tlc6zwQk2nPk/4f0cHkOdEixQPC/j8UtKDdITswvLAy1OZ1w==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "@types/mdast": "^4.0.0",
+        "devlop": "^1.0.0",
+        "longest-streak": "^3.0.0",
+        "mdast-util-from-markdown": "^2.0.0",
+        "mdast-util-to-markdown": "^2.1.0",
+        "unist-util-remove-position": "^5.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/mdast-util-math/node_modules/@types/hast": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz",
+      "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "*"
+      }
+    },
     "node_modules/mdast-util-mdx-expression": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.0.tgz",
@@ -6272,6 +6538,25 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/micromark-extension-math": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/micromark-extension-math/-/micromark-extension-math-3.1.0.tgz",
+      "integrity": "sha512-lvEqd+fHjATVs+2v/8kg9i5Q0AP2k85H0WUOwpIVvUML8BapsMvh1XAogmQjOCsLpoKRCVQqEkQBB3NhVBcsOg==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/katex": "^0.16.0",
+        "devlop": "^1.0.0",
+        "katex": "^0.16.0",
+        "micromark-factory-space": "^2.0.0",
+        "micromark-util-character": "^2.0.0",
+        "micromark-util-symbol": "^2.0.0",
+        "micromark-util-types": "^2.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
     "node_modules/micromark-factory-destination": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.0.tgz",
@@ -7058,6 +7343,18 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/parse5": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
+      "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
+      "license": "MIT",
+      "dependencies": {
+        "entities": "^4.4.0"
+      },
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      }
+    },
     "node_modules/path-exists": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
@@ -7371,9 +7668,9 @@
       }
     },
     "node_modules/prettier-plugin-tailwindcss": {
-      "version": "0.6.6",
-      "resolved": "https://registry.npmjs.org/prettier-plugin-tailwindcss/-/prettier-plugin-tailwindcss-0.6.6.tgz",
-      "integrity": "sha512-OPva5S7WAsPLEsOuOWXATi13QrCKACCiIonFgIR6V4lYv4QLp++UXVhZSzRbZxXGimkQtQT86CC6fQqTOybGng==",
+      "version": "0.6.8",
+      "resolved": "https://registry.npmjs.org/prettier-plugin-tailwindcss/-/prettier-plugin-tailwindcss-0.6.8.tgz",
+      "integrity": "sha512-dGu3kdm7SXPkiW4nzeWKCl3uoImdd5CTZEJGxyypEPL37Wj0HT2pLqjrvSei1nTeuQfO4PUfjeW5cTUNRLZ4sA==",
       "dev": true,
       "engines": {
         "node": ">=14.21.3"
@@ -7834,6 +8131,34 @@
         "url": "https://github.com/sponsors/mysticatea"
       }
     },
+    "node_modules/rehype-katex": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/rehype-katex/-/rehype-katex-7.0.1.tgz",
+      "integrity": "sha512-OiM2wrZ/wuhKkigASodFoo8wimG3H12LWQaH8qSPVJn9apWKFSH3YOCtbKpBorTVw/eI7cuT21XBbvwEswbIOA==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/hast": "^3.0.0",
+        "@types/katex": "^0.16.0",
+        "hast-util-from-html-isomorphic": "^2.0.0",
+        "hast-util-to-text": "^4.0.0",
+        "katex": "^0.16.0",
+        "unist-util-visit-parents": "^6.0.0",
+        "vfile": "^6.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/rehype-katex/node_modules/@types/hast": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz",
+      "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "*"
+      }
+    },
     "node_modules/remark-gfm": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.0.tgz",
@@ -7851,6 +8176,22 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/remark-math": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/remark-math/-/remark-math-6.0.0.tgz",
+      "integrity": "sha512-MMqgnP74Igy+S3WwnhQ7kqGlEerTETXMvJhrUzDikVZ2/uogJCb+WHUg97hK9/jcfc0dkD73s3LN8zU49cTEtA==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/mdast": "^4.0.0",
+        "mdast-util-math": "^3.0.0",
+        "micromark-extension-math": "^3.0.0",
+        "unified": "^11.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
     "node_modules/remark-parse": {
       "version": "11.0.0",
       "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz",
@@ -8915,16 +9256,16 @@
       }
     },
     "node_modules/typescript": {
-      "version": "4.9.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
-      "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+      "version": "5.6.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz",
+      "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==",
       "dev": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
       },
       "engines": {
-        "node": ">=4.2.0"
+        "node": ">=14.17"
       }
     },
     "node_modules/unbox-primitive": {
@@ -8960,6 +9301,20 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/unist-util-find-after": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz",
+      "integrity": "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "^3.0.0",
+        "unist-util-is": "^6.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
     "node_modules/unist-util-is": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz",
@@ -9103,6 +9458,20 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/vfile-location": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-5.0.3.tgz",
+      "integrity": "sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/unist": "^3.0.0",
+        "vfile": "^6.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
     "node_modules/vfile-message": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.2.tgz",
@@ -9198,6 +9567,16 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/web-namespaces": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz",
+      "integrity": "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==",
+      "license": "MIT",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
+      }
+    },
     "node_modules/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

frontend/package.json

@@ -20,7 +20,6 @@
   },
   "dependencies": {
     "@reduxjs/toolkit": "^2.2.7",
-    "@vercel/analytics": "^0.1.10",
     "chart.js": "^4.4.4",
     "i18next": "^23.15.1",
     "i18next-browser-languagedetector": "^8.0.0",
@@ -35,7 +34,9 @@
     "react-redux": "^8.0.5",
     "react-router-dom": "^6.8.1",
     "react-syntax-highlighter": "^15.5.0",
-    "remark-gfm": "^4.0.0"
+    "rehype-katex": "^7.0.1",
+    "remark-gfm": "^4.0.0",
+    "remark-math": "^6.0.0"
   },
   "devDependencies": {
     "@types/react": "^18.0.27",
@@ -53,14 +54,14 @@
     "eslint-plugin-prettier": "^5.2.1",
     "eslint-plugin-promise": "^6.6.0",
     "eslint-plugin-react": "^7.35.0",
-    "eslint-plugin-unused-imports": "^2.0.0",
+    "eslint-plugin-unused-imports": "^4.1.4",
     "husky": "^8.0.0",
     "lint-staged": "^15.2.10",
     "postcss": "^8.4.41",
     "prettier": "^3.3.3",
-    "prettier-plugin-tailwindcss": "^0.6.6",
+    "prettier-plugin-tailwindcss": "^0.6.8",
     "tailwindcss": "^3.4.11",
-    "typescript": "^4.9.5",
+    "typescript": "^5.6.2",
     "vite": "^5.4.6",
     "vite-plugin-svgr": "^4.2.0"
   }

frontend/src/App.tsx

@@ -3,7 +3,6 @@ import Navigation from './Navigation';
 import Conversation from './conversation/Conversation';
 import About from './About';
 import PageNotFound from './PageNotFound';
-import { inject } from '@vercel/analytics';
 import { useMediaQuery } from './hooks';
 import { useState } from 'react';
 import Setting from './settings';
@@ -11,7 +10,6 @@ import './locale/i18n';
 import { Outlet } from 'react-router-dom';
 import { SharedConversation } from './conversation/SharedConversation';
 import { useDarkTheme } from './hooks';
-inject();
 
 function MainLayout() {
   const { isMobile } = useMediaQuery();
@@ -21,7 +19,7 @@ function MainLayout() {
     <div className="dark:bg-raisin-black relative h-screen overflow-auto">
       <Navigation navOpen={navOpen} setNavOpen={setNavOpen} />
       <div
-        className={`h-[calc(100dvh-64px)] sm:h-screen ${
+        className={`h-[calc(100dvh-64px)] md:h-screen ${
           !isMobile
             ? `ml-0 ${!navOpen ? 'md:mx-auto lg:mx-auto' : 'md:ml-72'}`
             : 'ml-0 md:ml-16'
@@ -34,7 +32,10 @@ function MainLayout() {
 }
 
 export default function App() {
-  useDarkTheme();
+  const [, , componentMounted] = useDarkTheme();
+  if (!componentMounted) {
+    return <div />;
+  }
   return (
     <div className="h-full relative overflow-auto">
       <Routes>

frontend/src/Hero.tsx

@@ -37,7 +37,7 @@ export default function Hero({
               <Fragment key={key}>
                 <button
                   onClick={() => handleQuestion({ question: demo.query })}
-                  className="w-full rounded-full border-2 border-silver px-6 py-4 text-left hover:border-gray-4000 dark:hover:border-gray-3000 xl:min-w-[24vw]"
+                  className="w-full rounded-full border border-silver px-6 py-4 text-left hover:border-gray-4000 dark:hover:border-gray-3000 xl:min-w-[24vw]"
                 >
                   <p className="mb-1 font-semibold text-black dark:text-silver">
                     {demo.header}

frontend/src/Navigation.tsx

@@ -2,16 +2,15 @@ import { useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
 import { NavLink, useNavigate } from 'react-router-dom';
-
 import conversationService from './api/services/conversationService';
 import userService from './api/services/userService';
 import Add from './assets/add.svg';
+import openNewChat from './assets/openNewChat.svg';
+import Hamburger from './assets/hamburger.svg';
 import DocsGPT3 from './assets/cute_docsgpt3.svg';
 import Discord from './assets/discord.svg';
 import Expand from './assets/expand.svg';
 import Github from './assets/github.svg';
-import Hamburger from './assets/hamburger.svg';
-import Info from './assets/info.svg';
 import SettingGear from './assets/settingGear.svg';
 import Twitter from './assets/TwitterX.svg';
 import UploadIcon from './assets/upload.svg';
@@ -19,6 +18,7 @@ import SourceDropdown from './components/SourceDropdown';
 import {
   setConversation,
   updateConversationId,
+  handleAbort,
 } from './conversation/conversationSlice';
 import ConversationTile from './conversation/ConversationTile';
 import { useDarkTheme, useMediaQuery, useOutsideAlerter } from './hooks';
@@ -33,14 +33,19 @@ import {
   selectConversations,
   selectModalStateDeleteConv,
   selectSelectedDocs,
-  selectSelectedDocsStatus,
   selectSourceDocs,
+  selectPaginatedDocuments,
   setConversations,
   setModalStateDeleteConv,
   setSelectedDocs,
   setSourceDocs,
+  setPaginatedDocuments,
 } from './preferences/preferenceSlice';
+import Spinner from './assets/spinner.svg';
+import SpinnerDark from './assets/spinner-dark.svg';
+import { selectQueries } from './conversation/conversationSlice';
 import Upload from './upload/Upload';
+import Help from './components/Help';
 
 interface NavigationProps {
   navOpen: boolean;
@@ -63,25 +68,23 @@ NavImage.propTypes = {
 }; */
 export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   const dispatch = useDispatch();
+  const queries = useSelector(selectQueries);
   const docs = useSelector(selectSourceDocs);
   const selectedDocs = useSelector(selectSelectedDocs);
   const conversations = useSelector(selectConversations);
   const modalStateDeleteConv = useSelector(selectModalStateDeleteConv);
   const conversationId = useSelector(selectConversationId);
+  const paginatedDocuments = useSelector(selectPaginatedDocuments);
+  const [isDeletingConversation, setIsDeletingConversation] = useState(false);
 
   const { isMobile } = useMediaQuery();
   const [isDarkTheme] = useDarkTheme();
   const [isDocsListOpen, setIsDocsListOpen] = useState(false);
   const { t } = useTranslation();
-
   const isApiKeySet = useSelector(selectApiKeyStatus);
   const [apiKeyModalState, setApiKeyModalState] =
     useState<ActiveState>('INACTIVE');
 
-  const isSelectedDocsSet = useSelector(selectSelectedDocsStatus);
-  const [selectedDocsModalState, setSelectedDocsModalState] =
-    useState<ActiveState>(isSelectedDocsSet ? 'INACTIVE' : 'ACTIVE');
-
   const [uploadModalState, setUploadModalState] =
     useState<ActiveState>('INACTIVE');
 
@@ -90,22 +93,28 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   const navigate = useNavigate();
 
   useEffect(() => {
-    if (!conversations) {
+    if (!conversations?.data) {
       fetchConversations();
     }
-  }, [conversations, dispatch]);
+    if (queries.length === 0) {
+      resetConversation();
+    }
+  }, [conversations?.data, dispatch]);
 
   async function fetchConversations() {
+    dispatch(setConversations({ ...conversations, loading: true }));
     return await getConversations()
       .then((fetchedConversations) => {
         dispatch(setConversations(fetchedConversations));
       })
       .catch((error) => {
         console.error('Failed to fetch conversations: ', error);
+        dispatch(setConversations({ data: null, loading: false }));
       });
   }
 
   const handleDeleteAllConversations = () => {
+    setIsDeletingConversation(true);
     conversationService
       .deleteAll()
       .then(() => {
@@ -115,10 +124,12 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   };
 
   const handleDeleteConversation = (id: string) => {
+    setIsDeletingConversation(true);
     conversationService
       .delete(id, {})
       .then(() => {
         fetchConversations();
+        resetConversation();
       })
       .catch((error) => console.error(error));
   };
@@ -131,9 +142,18 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
       })
       .then((updatedDocs) => {
         dispatch(setSourceDocs(updatedDocs));
+        const updatedPaginatedDocs = paginatedDocuments?.filter(
+          (document) => document.id !== doc.id,
+        );
+        dispatch(
+          setPaginatedDocuments(updatedPaginatedDocs || paginatedDocuments),
+        );
         dispatch(
           setSelectedDocs(
-            updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'),
+            Array.isArray(updatedDocs) &&
+              updatedDocs?.find(
+                (doc: Doc) => doc.name.toLowerCase() === 'default',
+              ),
           ),
         );
       })
@@ -155,6 +175,20 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
       });
   };
 
+  const resetConversation = () => {
+    handleAbort();
+    dispatch(setConversation([]));
+    dispatch(
+      updateConversationId({
+        query: { conversationId: null },
+      }),
+    );
+  };
+  const newChat = () => {
+    if (queries && queries?.length > 0) {
+      resetConversation();
+    }
+  };
   async function updateConversationName(updatedConversation: {
     name: string;
     id: string;
@@ -187,23 +221,43 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
     setNavOpen(!isMobile);
   }, [isMobile]);
   useDefaultDocument();
+
   return (
     <>
       {!navOpen && (
-        <button
-          className="duration-25 absolute  top-3 left-3 z-20 hidden transition-all md:block"
-          onClick={() => {
-            setNavOpen(!navOpen);
-          }}
-        >
-          <img
-            src={Expand}
-            alt="menu toggle"
-            className={`${
-              !navOpen ? 'rotate-180' : 'rotate-0'
-            } m-auto transition-all duration-200`}
-          />
-        </button>
+        <div className="duration-25 absolute  top-3 left-3 z-20 hidden transition-all md:block">
+          <div className="flex gap-3 items-center">
+            <button
+              onClick={() => {
+                setNavOpen(!navOpen);
+              }}
+            >
+              <img
+                src={Expand}
+                alt="menu toggle"
+                className={`${
+                  !navOpen ? 'rotate-180' : 'rotate-0'
+                } m-auto transition-all duration-200`}
+              />
+            </button>
+            {queries?.length > 0 && (
+              <button
+                onClick={() => {
+                  newChat();
+                }}
+              >
+                <img
+                  src={openNewChat}
+                  alt="open new chat icon"
+                  className="cursor-pointer"
+                />
+              </button>
+            )}
+            <div className="text-[#949494] font-medium text-[20px]">
+              DocsGPT
+            </div>
+          </div>
+        </div>
       )}
       <div
         ref={navRef}
@@ -214,9 +268,18 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         <div
           className={'visible mt-2 flex h-[6vh] w-full justify-between md:h-12'}
         >
-          <div className="my-auto mx-4 flex cursor-pointer gap-1.5">
-            <img className="mb-2 h-10" src={DocsGPT3} alt="" />
-            <p className="my-auto text-2xl font-semibold">DocsGPT</p>
+          <div
+            className="my-auto mx-4 flex cursor-pointer gap-1.5"
+            onClick={() => {
+              if (isMobile) {
+                setNavOpen(!navOpen);
+              }
+            }}
+          >
+            <a href="/" className="flex gap-1.5">
+              <img className="mb-2 h-10" src={DocsGPT3} alt="" />
+              <p className="my-auto text-2xl font-semibold">DocsGPT</p>
+            </a>
           </div>
           <button
             className="float-right mr-5"
@@ -236,12 +299,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         <NavLink
           to={'/'}
           onClick={() => {
-            dispatch(setConversation([]));
-            dispatch(
-              updateConversationId({
-                query: { conversationId: null },
-              }),
-            );
+            if (isMobile) {
+              setNavOpen(!navOpen);
+            }
+            resetConversation();
           }}
           className={({ isActive }) =>
             `${
@@ -258,18 +319,35 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             {t('newChat')}
           </p>
         </NavLink>
-        <div className="mb-auto h-[78vh] overflow-y-auto overflow-x-hidden dark:text-white">
-          {conversations && conversations.length > 0 ? (
+        <div
+          id="conversationsMainDiv"
+          className="mb-auto h-[78vh] overflow-y-auto overflow-x-hidden dark:text-white"
+        >
+          {conversations?.loading && !isDeletingConversation && (
+            <div className="absolute top-1/2 left-1/2 transform -translate-x-1/2 -translate-y-1/2">
+              <img
+                src={isDarkTheme ? SpinnerDark : Spinner}
+                className="animate-spin cursor-pointer bg-transparent"
+                alt="Loading..."
+              />
+            </div>
+          )}
+          {conversations?.data && conversations.data.length > 0 ? (
             <div>
               <div className=" my-auto mx-4 mt-2 flex h-6 items-center justify-between gap-4 rounded-3xl">
                 <p className="mt-1 ml-4 text-sm font-semibold">{t('chats')}</p>
               </div>
               <div className="conversations-container">
-                {conversations?.map((conversation) => (
+                {conversations.data?.map((conversation) => (
                   <ConversationTile
                     key={conversation.id}
                     conversation={conversation}
                     selectConversation={(id) => handleConversationClick(id)}
+                    onCoversationClick={() => {
+                      if (isMobile) {
+                        setNavOpen(false);
+                      }
+                    }}
                     onDeleteConversation={(id) => handleDeleteConversation(id)}
                     onSave={(conversation) =>
                       updateConversationName(conversation)
@@ -282,7 +360,6 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             <></>
           )}
         </div>
-
         <div className="flex h-auto flex-col justify-end text-eerie-black dark:text-white">
           <div className="flex flex-col-reverse border-b-[1px] dark:border-b-purple-taupe">
             <div className="relative my-4 mx-4 flex gap-2">
@@ -293,17 +370,33 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                 isDocsListOpen={isDocsListOpen}
                 setIsDocsListOpen={setIsDocsListOpen}
                 handleDeleteClick={handleDeleteClick}
+                handlePostDocumentSelect={(option?: string) => {
+                  if (isMobile) {
+                    setNavOpen(!navOpen);
+                  }
+                }}
               />
               <img
                 className="mt-2 h-9 w-9 hover:cursor-pointer"
                 src={UploadIcon}
-                onClick={() => setUploadModalState('ACTIVE')}
+                onClick={() => {
+                  setUploadModalState('ACTIVE');
+                  if (isMobile) {
+                    setNavOpen(!navOpen);
+                  }
+                }}
               ></img>
             </div>
             <p className="ml-5 mt-3 text-sm font-semibold">{t('sourceDocs')}</p>
           </div>
           <div className="flex flex-col gap-2 border-b-[1px] py-2 dark:border-b-purple-taupe">
             <NavLink
+              onClick={() => {
+                if (isMobile) {
+                  setNavOpen(!navOpen);
+                }
+                resetConversation();
+              }}
               to="/settings"
               className={({ isActive }) =>
                 `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
@@ -321,77 +414,69 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
               </p>
             </NavLink>
           </div>
-          <div className="flex justify-between gap-2 border-b-[1.5px] py-2 dark:border-b-purple-taupe">
-            <NavLink
-              to="/about"
-              className={({ isActive }) =>
-                `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
-                  isActive ? 'bg-gray-3000 dark:bg-[#28292E]' : ''
-                }`
-              }
-            >
-              <img
-                src={Info}
-                alt="icon"
-                className="ml-2 w-5 filter dark:invert"
-              />
-              <p className="my-auto pr-1 text-sm">{t('about')}</p>
-            </NavLink>
-            <div className="flex items-center justify-evenly gap-1 px-1">
-              <NavLink
-                target="_blank"
-                to={'https://discord.gg/WHJdfbQDR4'}
-                className={
-                  'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                }
-              >
-                <img
-                  src={Discord}
-                  alt="discord"
-                  className="m-2 w-6 self-center filter dark:invert"
-                />
-              </NavLink>
-              <NavLink
-                target="_blank"
-                to={'https://twitter.com/docsgptai'}
-                className={
-                  'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                }
-              >
-                <img
-                  src={Twitter}
-                  alt="x"
-                  className="m-2 w-5 self-center filter dark:invert"
-                />
-              </NavLink>
-              <NavLink
-                target="_blank"
-                to={'https://github.com/arc53/docsgpt'}
-                className={
-                  'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                }
-              >
-                <img
-                  src={Github}
-                  alt="github"
-                  className="m-2 w-6 self-center filter dark:invert"
-                />
-              </NavLink>
+          <div className="flex flex-col justify-end text-eerie-black dark:text-white">
+            <div className="flex justify-between items-center py-1">
+              <Help />
+
+              <div className="flex items-center gap-1 pr-4">
+                <NavLink
+                  target="_blank"
+                  to={'https://discord.gg/WHJdfbQDR4'}
+                  className={
+                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
+                  }
+                >
+                  <img
+                    src={Discord}
+                    alt="discord"
+                    className="m-2 w-6 self-center filter dark:invert"
+                  />
+                </NavLink>
+                <NavLink
+                  target="_blank"
+                  to={'https://twitter.com/docsgptai'}
+                  className={
+                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
+                  }
+                >
+                  <img
+                    src={Twitter}
+                    alt="x"
+                    className="m-2 w-5 self-center filter dark:invert"
+                  />
+                </NavLink>
+                <NavLink
+                  target="_blank"
+                  to={'https://github.com/arc53/docsgpt'}
+                  className={
+                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
+                  }
+                >
+                  <img
+                    src={Github}
+                    alt="github"
+                    className="m-2 w-6 self-center filter dark:invert"
+                  />
+                </NavLink>
+              </div>
             </div>
           </div>
         </div>
       </div>
       <div className="sticky z-10 h-16 w-full border-b-2 bg-gray-50 dark:border-b-purple-taupe dark:bg-chinese-black md:hidden">
-        <button
-          className="mt-5 ml-6 h-6 w-6 md:hidden"
-          onClick={() => setNavOpen(true)}
-        >
-          <img
-            src={Hamburger}
-            alt="menu toggle"
-            className="w-7 filter dark:invert"
-          />
-        </button>
+        <div className="flex gap-6 items-center h-full ml-6 ">
+          <button
+            className=" h-6 w-6 md:hidden"
+            onClick={() => setNavOpen(true)}
+          >
+            <img
+              src={Hamburger}
+              alt="menu toggle"
+              className="w-7 filter dark:invert"
+            />
+          </button>
+          <div className="text-[#949494] font-medium text-[20px]">DocsGPT</div>
+        </div>
       </div>
       <APIKeyModal
         modalState={apiKeyModalState}
@@ -403,10 +488,13 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         setModalState={setModalStateDeleteConv}
         handleDeleteAllConv={handleDeleteAllConversations}
       />
-      <Upload
-        modalState={uploadModalState}
-        setModalState={setUploadModalState}
-      ></Upload>
+      {uploadModalState === 'ACTIVE' && (
+        <Upload
+          setModalState={setUploadModalState}
+          isOnboarding={false}
+          close={() => setUploadModalState('INACTIVE')}
+        ></Upload>
+      )}
     </>
   );
 }

frontend/src/api/endpoints.ts

@@ -1,7 +1,8 @@
 const endpoints = {
   USER: {
-    DOCS: '/api/combine',
+    DOCS: '/api/sources',
     DOCS_CHECK: '/api/docs_check',
+    DOCS_PAGINATED: '/api/sources/paginated',
     API_KEYS: '/api/get_api_keys',
     CREATE_API_KEY: '/api/create_api_key',
     DELETE_API_KEY: '/api/delete_api_key',

frontend/src/api/services/userService.ts

@@ -2,7 +2,9 @@ import apiClient from '../client';
 import endpoints from '../endpoints';
 
 const userService = {
-  getDocs: (): Promise<any> => apiClient.get(endpoints.USER.DOCS),
+  getDocs: (): Promise<any> => apiClient.get(`${endpoints.USER.DOCS}`),
+  getDocsWithPagination: (query: string): Promise<any> =>
+    apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`),
   checkDocs: (data: any): Promise<any> =>
     apiClient.post(endpoints.USER.DOCS_CHECK, data),
   getAPIKeys: (): Promise<any> => apiClient.get(endpoints.USER.API_KEYS),

frontend/src/assets/Loading.svg

@@ -0,0 +1,3 @@
+<svg width="18" height="18" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M9 3.66667V1M12.7778 5.22222L14.6889 3.31111M14.3333 9H17M12.7778 12.7778L14.6889 14.6889M9 14.3333V17M5.22222 12.7778L3.31111 14.6889M3.66667 9H1M5.22222 5.22222L3.31111 3.31111" stroke="#949494" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

frontend/src/assets/caret-sort.svg

@@ -0,0 +1 @@
+<svg width="18" height="18" viewBox="0 0 15 15" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M4.93179 5.43179C4.75605 5.60753 4.75605 5.89245 4.93179 6.06819C5.10753 6.24392 5.39245 6.24392 5.56819 6.06819L7.49999 4.13638L9.43179 6.06819C9.60753 6.24392 9.89245 6.24392 10.0682 6.06819C10.2439 5.89245 10.2439 5.60753 10.0682 5.43179L7.81819 3.18179C7.73379 3.0974 7.61933 3.04999 7.49999 3.04999C7.38064 3.04999 7.26618 3.0974 7.18179 3.18179L4.93179 5.43179ZM10.0682 9.56819C10.2439 9.39245 10.2439 9.10753 10.0682 8.93179C9.89245 8.75606 9.60753 8.75606 9.43179 8.93179L7.49999 10.8636L5.56819 8.93179C5.39245 8.75606 5.10753 8.75606 4.93179 8.93179C4.75605 9.10753 4.75605 9.39245 4.93179 9.56819L7.18179 11.8182C7.35753 11.9939 7.64245 11.9939 7.81819 11.8182L10.0682 9.56819Z" fill="gray" fill-rule="evenodd" clip-rule="evenodd"></path></svg>

frontend/src/assets/discord.svg

@@ -1 +1,3 @@
-<svg fill="none" width="20" height="20" viewBox="0 0 22 20" xmlns="http://www.w3.org/2000/svg" xml:space="preserve"><path d="M18.942 5.556a16.299 16.299 0 0 0-4.126-1.297c-.178.321-.385.754-.529 1.097a15.175 15.175 0 0 0-4.573 0 11.583 11.583 0 0 0-.535-1.097 16.274 16.274 0 0 0-4.129 1.3c-2.611 3.946-3.319 7.794-2.965 11.587a16.494 16.494 0 0 0 5.061 2.593 12.65 12.65 0 0 0 1.084-1.785 10.689 10.689 0 0 1-1.707-.831c.143-.106.283-.217.418-.331 3.291 1.539 6.866 1.539 10.118 0 .137.114.277.225.418.331-.541.326-1.114.606-1.71.832a12.52 12.52 0 0 0 1.084 1.785 16.46 16.46 0 0 0 5.064-2.595c.415-4.396-.709-8.209-2.973-11.589zM8.678 14.813c-.988 0-1.798-.922-1.798-2.045s.793-2.047 1.798-2.047 1.815.922 1.798 2.047c.001 1.123-.793 2.045-1.798 2.045zm6.644 0c-.988 0-1.798-.922-1.798-2.045s.793-2.047 1.798-2.047 1.815.922 1.798 2.047c0 1.123-.793 2.045-1.798 2.045z" fill="black" fill-opacity="0.54"/></svg>
+<svg width="24" height="25" viewBox="0 0 24 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M18.7359 5.33456C17.4059 4.70233 15.9659 4.24345 14.4659 3.97832C14.4528 3.97789 14.4397 3.98041 14.4276 3.98569C14.4155 3.99098 14.4047 3.9989 14.3959 4.00891C14.2159 4.34542 14.0059 4.78391 13.8659 5.12042C12.2749 4.87568 10.6569 4.87568 9.06594 5.12042C8.92594 4.77371 8.71594 4.34542 8.52594 4.00891C8.51594 3.98852 8.48594 3.97832 8.45594 3.97832C6.95594 4.24345 5.52594 4.70233 4.18594 5.33456C4.17594 5.33456 4.16594 5.34476 4.15594 5.35495C1.43594 9.50526 0.685937 13.5434 1.05594 17.5407C1.05594 17.5611 1.06594 17.5815 1.08594 17.5917C2.88594 18.9378 4.61594 19.7535 6.32594 20.294C6.35594 20.3042 6.38594 20.294 6.39594 20.2736C6.79594 19.7128 7.15594 19.1213 7.46594 18.4993C7.48594 18.4585 7.46594 18.4177 7.42594 18.4075C6.85594 18.1832 6.31594 17.918 5.78594 17.6121C5.74594 17.5917 5.74594 17.5305 5.77594 17.4999C5.88594 17.4184 5.99594 17.3266 6.10594 17.245C6.12594 17.2246 6.15594 17.2246 6.17594 17.2348C9.61594 18.8358 13.3259 18.8358 16.7259 17.2348C16.7459 17.2246 16.7759 17.2246 16.7959 17.245C16.9059 17.3368 17.0159 17.4184 17.1259 17.5101C17.1659 17.5407 17.1659 17.6019 17.1159 17.6223C16.5959 17.9384 16.0459 18.1934 15.4759 18.4177C15.4359 18.4279 15.4259 18.4789 15.4359 18.5095C15.7559 19.1315 16.1159 19.723 16.5059 20.2838C16.5359 20.294 16.5659 20.3042 16.5959 20.294C18.3159 19.7535 20.0459 18.9378 21.8459 17.5917C21.8659 17.5815 21.8759 17.5611 21.8759 17.5407C22.3159 12.9214 21.1459 8.91381 18.7759 5.35495C18.7659 5.34476 18.7559 5.33456 18.7359 5.33456ZM7.98594 15.1036C6.95594 15.1036 6.09594 14.1348 6.09594 12.9417C6.09594 11.7487 6.93594 10.7799 7.98594 10.7799C9.04594 10.7799 9.88594 11.7589 9.87594 12.9417C9.87594 14.1348 9.03594 15.1036 7.98594 15.1036ZM14.9559 15.1036C13.9259 15.1036 13.0659 14.1348 13.0659 12.9417C13.0659 11.7487 13.9059 10.7799 14.9559 10.7799C16.0159 10.7799 16.8559 11.7589 16.8459 12.9417C16.8459 14.1348 16.0159 15.1036 14.9559 15.1036Z" fill="#747474"/>
+</svg>

frontend/src/assets/documentation-dark.svg

@@ -1,3 +1,4 @@
-<svg xmlns="http://www.w3.org/2000/svg" x="0px" y="0px" width="100" height="100" viewBox="0,0,256,256">
-    <g transform="translate(-19.2,-19.2) scale(1.15,1.15)"><g fill="#949494" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M13.172,2h-7.172c-1.1,0 -2,0.9 -2,2v16c0,1.1 0.9,2 2,2h12c1.1,0 2,-0.9 2,-2v-11.172c0,-0.53 -0.211,-1.039 -0.586,-1.414l-4.828,-4.828c-0.375,-0.375 -0.884,-0.586 -1.414,-0.586zM15,18h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM15,14h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM13,9v-5.5l5.5,5.5z"></path></g></g></g>
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16.25 8.64258V16.25C16.25 16.7473 16.0525 17.2242 15.7008 17.5758C15.3492 17.9275 14.8723 18.125 14.375 18.125H5.625C5.12772 18.125 4.65081 17.9275 4.29917 17.5758C3.94754 17.2242 3.75 16.7473 3.75 16.25V3.75C3.75 3.25272 3.94754 2.77581 4.29917 2.42417C4.65081 2.07254 5.12772 1.875 5.625 1.875H9.48242C9.81383 1.87505 10.1316 2.0067 10.366 2.24102L15.884 7.75898C16.1183 7.99335 16.2499 8.31117 16.25 8.64258Z" stroke="#ECECF1" stroke-width="1.11111" stroke-linejoin="round"/>
+<path d="M10 2.1875V6.875C10 7.20652 10.1317 7.52446 10.3661 7.75888C10.6005 7.9933 10.9185 8.125 11.25 8.125H15.9375M6.875 11.25H13.125M6.875 14.375H13.125" stroke="#ECECF1" stroke-width="1.11111" stroke-linecap="round" stroke-linejoin="round"/>
 </svg>

frontend/src/assets/documentation.svg

@@ -1,3 +1,4 @@
-<svg xmlns="http://www.w3.org/2000/svg" x="0px" y="0px" width="100" height="100" viewBox="0,0,256,256">
-    <g transform="translate(-19.2,-19.2) scale(1.15,1.15)"><g fill="black" fill-opacity="0.54" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M13.172,2h-7.172c-1.1,0 -2,0.9 -2,2v16c0,1.1 0.9,2 2,2h12c1.1,0 2,-0.9 2,-2v-11.172c0,-0.53 -0.211,-1.039 -0.586,-1.414l-4.828,-4.828c-0.375,-0.375 -0.884,-0.586 -1.414,-0.586zM15,18h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM15,14h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM13,9v-5.5l5.5,5.5z"></path></g></g></g>
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16.25 8.64258V16.25C16.25 16.7473 16.0525 17.2242 15.7008 17.5758C15.3492 17.9275 14.8723 18.125 14.375 18.125H5.625C5.12772 18.125 4.65081 17.9275 4.29917 17.5758C3.94754 17.2242 3.75 16.7473 3.75 16.25V3.75C3.75 3.25272 3.94754 2.77581 4.29917 2.42417C4.65081 2.07254 5.12772 1.875 5.625 1.875H9.48242C9.81383 1.87505 10.1316 2.0067 10.366 2.24102L15.884 7.75898C16.1183 7.99335 16.2499 8.31117 16.25 8.64258Z" stroke="#6E6E6E" stroke-width="1.11111" stroke-linejoin="round"/>
+<path d="M10 2.1875V6.875C10 7.20652 10.1317 7.52446 10.3661 7.75888C10.6005 7.9933 10.9185 8.125 11.25 8.125H15.9375M6.875 11.25H13.125M6.875 14.375H13.125" stroke="#6E6E6E" stroke-width="1.11111" stroke-linecap="round" stroke-linejoin="round"/>
 </svg>

frontend/src/assets/double-arrow-left.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="12" viewBox="0 0 16 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M7.41 10.59L2.83 6L7.41 1.41L6 0L0 6L6 12L7.41 10.59Z" fill="black" fill-opacity="0.54" />
+    <path d="M15.41 10.59L10.83 6L15.41 1.41L14 0L8 6L14 12L15.41 10.59Z" fill="black"
+        fill-opacity="0.54" />
+</svg>

frontend/src/assets/double-arrow-right.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="12" viewBox="0 0 16 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M8.59 10.59L13.17 6L8.59 1.41L10 0L16 6L10 12L8.59 10.59Z" fill="black"
+        fill-opacity="0.54" />
+    <path d="M0.59 10.59L5.17 6L0.59 1.41L2 0L8 6L2 12L0.59 10.59Z" fill="black" fill-opacity="0.54" />
+</svg>

frontend/src/assets/envelope-dark.svg

@@ -0,0 +1,3 @@
+<svg width="18" height="18" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M3.04867 15.49H15.1775C16.5339 15.49 17.3172 14.7067 17.3172 13.1548V4.83755C17.3172 3.29309 16.5265 2.50977 14.9515 2.50977H2.82302C1.47463 2.50977 0.683594 3.28569 0.683594 4.83755V13.1545C0.683594 14.7138 1.48138 15.49 3.04867 15.49ZM8.10441 9.3803L2.41609 3.76816C2.58163 3.70034 2.77738 3.66273 3.01106 3.66273H14.9971C15.2305 3.66273 15.434 3.70034 15.6072 3.78327L9.92692 9.38062C9.60291 9.7043 9.31652 9.84766 9.01534 9.84766C8.71384 9.84766 8.42841 9.70398 8.10441 9.3803ZM1.83559 13.1548V4.76234L6.16717 9.01162L1.84331 13.2824C1.83592 13.2448 1.83559 13.1998 1.83559 13.1548ZM16.1646 4.84494V13.2599L11.8629 9.0113L16.1649 4.78516L16.1646 4.84494ZM3.01106 14.3377C2.79249 14.3377 2.61184 14.3075 2.4537 14.2397L6.95852 9.78723L7.44838 10.2694C7.97552 10.7891 8.48017 11.0077 9.01534 11.0077C9.54249 11.0077 10.0548 10.7891 10.5823 10.2694L11.0718 9.78723L15.5696 14.2319C15.4111 14.3075 15.2154 14.3374 14.9968 14.3374L3.01106 14.3377Z" fill="#ECECF1"/>
+</svg>

frontend/src/assets/envelope.svg

@@ -0,0 +1,3 @@
+<svg width="18" height="18" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M3.04867 15.49H15.1775C16.5339 15.49 17.3172 14.7067 17.3172 13.1548V4.83755C17.3172 3.29309 16.5265 2.50977 14.9515 2.50977H2.82302C1.47463 2.50977 0.683594 3.28569 0.683594 4.83755V13.1545C0.683594 14.7138 1.48138 15.49 3.04867 15.49ZM8.10441 9.3803L2.41609 3.76816C2.58163 3.70034 2.77738 3.66273 3.01106 3.66273H14.9971C15.2305 3.66273 15.434 3.70034 15.6072 3.78327L9.92692 9.38062C9.60291 9.7043 9.31652 9.84766 9.01534 9.84766C8.71384 9.84766 8.42841 9.70398 8.10441 9.3803ZM1.83559 13.1548V4.76234L6.16717 9.01162L1.84331 13.2824C1.83592 13.2448 1.83559 13.1998 1.83559 13.1548ZM16.1646 4.84494V13.2599L11.8629 9.0113L16.1649 4.78516L16.1646 4.84494ZM3.01106 14.3377C2.79249 14.3377 2.61184 14.3075 2.4537 14.2397L6.95852 9.78723L7.44838 10.2694C7.97552 10.7891 8.48017 11.0077 9.01534 11.0077C9.54249 11.0077 10.0548 10.7891 10.5823 10.2694L11.0718 9.78723L15.5696 14.2319C15.4111 14.3075 15.2154 14.3374 14.9968 14.3374L3.01106 14.3377Z" fill="#6F6F6F"/>
+</svg>

frontend/src/assets/file_upload.svg

@@ -0,0 +1,3 @@
+<svg width="28" height="34" viewBox="0 0 28 34" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M10 26.0003H18C19.1 26.0003 20 25.1003 20 24.0003V14.0003H23.18C24.96 14.0003 25.86 11.8403 24.6 10.5803L15.42 1.40032C15.235 1.21491 15.0152 1.06782 14.7732 0.967453C14.5313 0.86709 14.2719 0.81543 14.01 0.81543C13.7481 0.81543 13.4887 0.86709 13.2468 0.967453C13.0048 1.06782 12.785 1.21491 12.6 1.40032L3.42 10.5803C2.16 11.8403 3.04 14.0003 4.82 14.0003H8V24.0003C8 25.1003 8.9 26.0003 10 26.0003ZM2 30.0003H26C27.1 30.0003 28 30.9003 28 32.0003C28 33.1003 27.1 34.0003 26 34.0003H2C0.9 34.0003 0 33.1003 0 32.0003C0 30.9003 0.9 30.0003 2 30.0003Z" fill="#949494"/>
+</svg>

frontend/src/assets/openNewChat.svg

@@ -0,0 +1,4 @@
+<svg width="21" height="20" viewBox="0 0 21 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="1.66663" y="1" width="18" height="18" rx="9" stroke="#747474" stroke-width="2"/>
+<path d="M9.89252 14.1268C9.89252 14.3321 9.97405 14.5289 10.1192 14.674C10.2643 14.8191 10.4611 14.9006 10.6663 14.9006C10.8716 14.9006 11.0684 14.8191 11.2135 14.674C11.3586 14.5289 11.4401 14.3321 11.4401 14.1268L11.4401 10.7737L14.7933 10.7737C14.9985 10.7737 15.1953 10.6922 15.3404 10.547C15.4856 10.4019 15.5671 10.2051 15.5671 9.99989C15.5671 9.79466 15.4856 9.59784 15.3404 9.45272C15.1953 9.30761 14.9985 9.22608 14.7933 9.22608L11.4401 9.22608L11.4401 5.87293C11.4401 5.6677 11.3586 5.47088 11.2135 5.32576C11.0684 5.18065 10.8716 5.09912 10.6663 5.09912C10.4611 5.09912 10.2643 5.18065 10.1192 5.32576C9.97405 5.47088 9.89252 5.6677 9.89252 5.87293L9.89252 9.22608L6.53937 9.22608C6.33414 9.22608 6.13732 9.30761 5.99221 9.45272C5.84709 9.59784 5.76556 9.79466 5.76556 9.99989C5.76556 10.2051 5.84709 10.4019 5.99221 10.547C6.13732 10.6922 6.33414 10.7737 6.53937 10.7737L9.89252 10.7737L9.89252 14.1268Z" fill="#747474"/>
+</svg>

frontend/src/assets/single-left-arrow.svg

@@ -0,0 +1,3 @@
+<svg width="8" height="12" viewBox="0 0 8 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M7.41 10.59L2.83 6L7.41 1.41L6 0L0 6L6 12L7.41 10.59Z" fill="black" fill-opacity="0.54" />
+</svg>

frontend/src/assets/single-right-arrow.svg

@@ -0,0 +1,3 @@
+<svg width="8" height="12" viewBox="0 0 8 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M0.59 1.41L5.17 6L0.59 10.59L2 12L8 6L2 0L0.59 1.41Z" fill="black" fill-opacity="0.54" />
+</svg>

frontend/src/assets/speaker.svg

@@ -0,0 +1,4 @@
+<svg width="18" height="18" viewBox="0 0 21 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M1 9.59652V6.40372C1 5.94773 1.18114 5.51041 1.50358 5.18797C1.82602 4.86553 2.26334 4.68439 2.71933 4.68439H5.21237C5.38045 4.68435 5.54483 4.63503 5.68518 4.54254L10.8432 1.1417C10.9728 1.05636 11.1231 1.00768 11.2781 1.00084C11.4331 0.993993 11.5871 1.02924 11.7237 1.10283C11.8603 1.17643 11.9745 1.28563 12.054 1.41885C12.1336 1.55207 12.1756 1.70435 12.1757 1.85952V14.1407C12.1756 14.2959 12.1336 14.4482 12.054 14.5814C11.9745 14.7146 11.8603 14.8238 11.7237 14.8974C11.5871 14.971 11.4331 15.0063 11.2781 14.9994C11.1231 14.9926 10.9728 14.9439 10.8432 14.8585L5.68518 11.4577C5.54483 11.3652 5.38045 11.3159 5.21237 11.3159H2.71933C2.26334 11.3159 1.82602 11.1347 1.50358 10.8123C1.18114 10.4898 1 10.0525 1 9.59652Z" stroke="#949494" stroke-width="1.4"/>
+<path d="M15.1846 4.13149C15.1846 4.13149 16.4741 5.42099 16.4741 7.57016C16.4741 9.71932 15.1846 11.0088 15.1846 11.0088M17.7636 1.55249C17.7636 1.55249 19.9127 3.70166 19.9127 7.57016C19.9127 11.4387 17.7636 13.5878 17.7636 13.5878" stroke="#949494" stroke-width="1.4" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

frontend/src/assets/stopspeech.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M8 1.14286C9.35622 1.14286 10.682 1.54502 11.8096 2.29849C12.9373 3.05197 13.8162 4.1229 14.3352 5.37588C14.8542 6.62886 14.99 8.0076 14.7254 9.33776C14.4608 10.6679 13.8077 11.8897 12.8487 12.8487C11.8897 13.8077 10.6679 14.4608 9.33776 14.7254C8.00761 14.99 6.62887 14.8542 5.37589 14.3352C4.12291 13.8162 3.05197 12.9373 2.2985 11.8096C1.54502 10.682 1.14286 9.35621 1.14286 8C1.14286 6.18137 1.86531 4.43723 3.15127 3.15127C4.43723 1.8653 6.18137 1.14286 8 1.14286ZM8 0C6.41775 0 4.87103 0.469192 3.55544 1.34824C2.23985 2.22729 1.21447 3.47672 0.608967 4.93853C0.00346626 6.40034 -0.15496 8.00887 0.153721 9.56072C0.462403 11.1126 1.22433 12.538 2.34315 13.6568C3.46197 14.7757 4.88743 15.5376 6.43928 15.8463C7.99113 16.155 9.59966 15.9965 11.0615 15.391C12.5233 14.7855 13.7727 13.7601 14.6518 12.4446C15.5308 11.129 16 9.58225 16 8C16 5.87827 15.1571 3.84344 13.6569 2.34314C12.1566 0.842854 10.1217 0 8 0Z" fill="#949494"/>
+<path d="M10.2857 5.71439V10.2858H5.71427V5.71439H10.2857ZM10.2857 4.57153H5.71427C5.41116 4.57153 5.12047 4.69194 4.90615 4.90627C4.69182 5.1206 4.57141 5.41129 4.57141 5.71439V10.2858C4.57141 10.5889 4.69182 10.8796 4.90615 11.0939C5.12047 11.3083 5.41116 11.4287 5.71427 11.4287H10.2857C10.5888 11.4287 10.8795 11.3083 11.0938 11.0939C11.3081 10.8796 11.4286 10.5889 11.4286 10.2858V5.71439C11.4286 5.41129 11.3081 5.1206 11.0938 4.90627C10.8795 4.69194 10.5888 4.57153 10.2857 4.57153Z" fill="#949494"/>
+</svg>
+

frontend/src/assets/website_collect.svg

@@ -0,0 +1,6 @@
+<svg width="58" height="49" viewBox="0 0 58 49" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M33.0002 45.3337H6.3335C5.27263 45.3337 4.25521 44.9122 3.50507 44.1621C2.75492 43.4119 2.3335 42.3945 2.3335 41.3337V6.66699C2.3335 5.60613 2.75492 4.58871 3.50507 3.83856C4.25521 3.08842 5.27263 2.66699 6.3335 2.66699H51.6668C52.7277 2.66699 53.7451 3.08842 54.4953 3.83856C55.2454 4.58871 55.6668 5.60613 55.6668 6.66699V24.0003M42.3335 40.0003L49.0002 46.667M49.0002 46.667L55.6668 40.0003M49.0002 46.667V30.667" stroke="#949494" stroke-width="4" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M2.3335 6.66699C2.3335 5.60613 2.75492 4.58871 3.50507 3.83856C4.25521 3.08842 5.27263 2.66699 6.3335 2.66699H51.6668C52.7277 2.66699 53.7451 3.08842 54.4953 3.83856C55.2454 4.58871 55.6668 5.60613 55.6668 6.66699V18.667H2.3335V6.66699Z" stroke="#949494" stroke-width="4"/>
+<path d="M7.66667 10.6673C7.66667 9.19456 8.86057 8.00065 10.3333 8.00065C11.8061 8.00065 13 9.19456 13 10.6673C13 12.1401 11.8061 13.334 10.3333 13.334C8.86057 13.334 7.66667 12.1401 7.66667 10.6673Z" fill="#949494"/>
+<path d="M15.6667 10.6673C15.6667 9.19456 16.8606 8.00065 18.3333 8.00065C19.8061 8.00065 21 9.19456 21 10.6673C21 12.1401 19.8061 13.334 18.3333 13.334C16.8606 13.334 15.6667 12.1401 15.6667 10.6673Z" fill="#949494"/>
+</svg>

frontend/src/components/DocumentPagination.tsx

@@ -0,0 +1,119 @@
+import React, { useState } from 'react';
+import SingleArrowLeft from '../assets/single-left-arrow.svg';
+import SingleArrowRight from '../assets/single-right-arrow.svg';
+import DoubleArrowLeft from '../assets/double-arrow-left.svg';
+import DoubleArrowRight from '../assets/double-arrow-right.svg';
+
+interface PaginationProps {
+  currentPage: number;
+  totalPages: number;
+  rowsPerPage: number;
+  onPageChange: (page: number) => void;
+  onRowsPerPageChange: (rows: number) => void;
+}
+
+const Pagination: React.FC<PaginationProps> = ({
+  currentPage,
+  totalPages,
+  rowsPerPage,
+  onPageChange,
+  onRowsPerPageChange,
+}) => {
+  const [rowsPerPageOptions] = useState([5, 10, 15, 20]);
+
+  const handlePreviousPage = () => {
+    if (currentPage > 1) {
+      onPageChange(currentPage - 1);
+    }
+  };
+
+  const handleNextPage = () => {
+    if (currentPage < totalPages) {
+      onPageChange(currentPage + 1);
+    }
+  };
+
+  const handleFirstPage = () => {
+    onPageChange(1);
+  };
+
+  const handleLastPage = () => {
+    onPageChange(totalPages);
+  };
+
+  return (
+    <div className="flex items-center text-xs justify-end gap-4 mt-2 p-2 border-gray-200">
+      <div className="flex items-center gap-2 ">
+        <span className="text-gray-900 dark:text-gray-50">Rows per page:</span>
+        <select
+          value={rowsPerPage}
+          onChange={(e) => onRowsPerPageChange(Number(e.target.value))}
+          className="border border-gray-300 rounded px-2 py-1 dark:bg-dark-charcoal dark:text-gray-50"
+        >
+          {rowsPerPageOptions.map((option) => (
+            <option
+              className="bg-white dark:bg-dark-charcoal dark:text-gray-50"
+              key={option}
+              value={option}
+            >
+              {option}
+            </option>
+          ))}
+        </select>
+      </div>
+
+      <div className="text-gray-900 dark:text-gray-50">
+        Page {currentPage} of {totalPages}
+      </div>
+
+      <div className="flex items-center gap-2 text-gray-900 dark:text-gray-50">
+        <button
+          onClick={handleFirstPage}
+          disabled={currentPage === 1}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={DoubleArrowLeft}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+        <button
+          onClick={handlePreviousPage}
+          disabled={currentPage === 1}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={SingleArrowLeft}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+        <button
+          onClick={handleNextPage}
+          disabled={currentPage === totalPages}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={SingleArrowRight}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+        <button
+          onClick={handleLastPage}
+          disabled={currentPage === totalPages}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={DoubleArrowRight}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+      </div>
+    </div>
+  );
+};
+
+export default Pagination;

frontend/src/components/Dropdown.tsx

@@ -140,7 +140,7 @@ function Dropdown({
                             : option.description
                         }`}
               </span>
-              {showEdit && onEdit && (
+              {showEdit && onEdit && option.type !== 'public' && (
                 <img
                   src={Edit}
                   alt="Edit"

frontend/src/components/Help.tsx

@@ -0,0 +1,80 @@
+import { useState, useRef, useEffect } from 'react';
+import Info from '../assets/info.svg';
+import PageIcon from '../assets/documentation.svg';
+import EmailIcon from '../assets/envelope.svg';
+import { useTranslation } from 'react-i18next';
+const Help = () => {
+  const [isOpen, setIsOpen] = useState(false);
+  const dropdownRef = useRef<HTMLDivElement | null>(null);
+  const buttonRef = useRef<HTMLButtonElement | null>(null);
+  const { t } = useTranslation();
+
+  const toggleDropdown = () => {
+    setIsOpen((prev) => !prev);
+  };
+
+  const handleClickOutside = (event: MouseEvent) => {
+    if (
+      dropdownRef.current &&
+      !dropdownRef.current.contains(event.target as Node) &&
+      buttonRef.current &&
+      !buttonRef.current.contains(event.target as Node)
+    ) {
+      setIsOpen(false);
+    }
+  };
+
+  useEffect(() => {
+    document.addEventListener('mousedown', handleClickOutside);
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+    };
+  }, []);
+
+  return (
+    <div className="relative inline-block text-sm" ref={dropdownRef}>
+      <button
+        ref={buttonRef}
+        onClick={toggleDropdown}
+        className="my-auto mx-4 w-full flex items-center h-9 gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E]"
+      >
+        <img src={Info} alt="info" className="ml-2 w-5 filter dark:invert" />
+        {t('help')}
+      </button>
+      {isOpen && (
+        <div
+          className={`absolute translate-x-4 -translate-y-28 z-10 w-48 shadow-lg bg-white dark:bg-[#444654] rounded-xl`}
+        >
+          <a
+            href="https://docs.docsgpt.cloud/"
+            target="_blank"
+            rel="noopener noreferrer"
+            className="flex items-start gap-4 px-4 py-2 text-black dark:text-white hover:bg-bright-gray dark:hover:bg-[#545561] rounded-t-xl"
+          >
+            <img
+              src={PageIcon}
+              alt="Documentation"
+              className="filter dark:invert"
+              width={20}
+            />
+            {t('documentation')}
+          </a>
+          <a
+            href="mailto:support@docsgpt.cloud"
+            className="flex items-start gap-4 px-4 py-2 text-black dark:text-white hover:bg-bright-gray dark:hover:bg-[#545561] rounded-b-xl"
+          >
+            <img
+              src={EmailIcon}
+              alt="Email Us"
+              className="filter dark:invert p-0.5"
+              width={20}
+            />
+            {t('emailUs')}
+          </a>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default Help;

frontend/src/components/RetryIcon.tsx

@@ -4,10 +4,11 @@ const RetryIcon = (props: SVGProps<SVGSVGElement>) => (
   <svg
     xmlns="http://www.w3.org/2000/svg"
     xmlSpace="preserve"
-    width={16}
-    height={16}
+    width={props.width ? props.width : 16}
+    height={props.height ? props.height : 16}
     fill={props.fill}
-    stroke={props.stroke}
+    stroke={props.stroke ? props.stroke : 'none'}
+    strokeWidth={props.strokeWidth ? props.strokeWidth : 10}
     viewBox="0 0 383.748 383.748"
     {...props}
   >

frontend/src/components/SettingsBar.tsx

@@ -0,0 +1,96 @@
+import React, { useCallback, useRef, useState } from 'react';
+import ArrowLeft from '../assets/arrow-left.svg';
+import ArrowRight from '../assets/arrow-right.svg';
+import { useTranslation } from 'react-i18next';
+
+type HiddenGradientType = 'left' | 'right' | undefined;
+
+const useTabs = () => {
+  const { t } = useTranslation();
+  const tabs = [
+    t('settings.general.label'),
+    t('settings.documents.label'),
+    t('settings.apiKeys.label'),
+    t('settings.analytics.label'),
+    t('settings.logs.label'),
+  ];
+  return tabs;
+};
+
+interface SettingsBarProps {
+  setActiveTab: React.Dispatch<React.SetStateAction<string>>;
+  activeTab: string;
+}
+
+const SettingsBar = ({ setActiveTab, activeTab }: SettingsBarProps) => {
+  const [hiddenGradient, setHiddenGradient] =
+    useState<HiddenGradientType>('left');
+  const containerRef = useRef<null | HTMLDivElement>(null);
+  const tabs = useTabs();
+  const scrollTabs = useCallback(
+    (direction: number) => {
+      if (containerRef.current) {
+        const container = containerRef.current;
+        container.scrollLeft += direction * 100; // Adjust the scroll amount as needed
+        if (container.scrollLeft === 0) {
+          setHiddenGradient('left');
+        } else if (
+          container.scrollLeft + container.offsetWidth ===
+          container.scrollWidth
+        ) {
+          setHiddenGradient('right');
+        } else {
+          setHiddenGradient(undefined);
+        }
+      }
+    },
+    [containerRef.current],
+  );
+  return (
+    <div className="relative mt-6 flex flex-row items-center space-x-1 md:space-x-0 overflow-auto">
+      <div
+        className={`${hiddenGradient === 'left' ? 'hidden' : ''} md:hidden absolute inset-y-0 left-6 w-14 bg-gradient-to-r from-white dark:from-raisin-black pointer-events-none`}
+      ></div>
+      <div
+        className={`${hiddenGradient === 'right' ? 'hidden' : ''} md:hidden absolute inset-y-0 right-6 w-14 bg-gradient-to-l from-white dark:from-raisin-black pointer-events-none`}
+      ></div>
+
+      <div className="md:hidden z-10">
+        <button
+          onClick={() => scrollTabs(-1)}
+          className="flex h-6 w-6 items-center rounded-full justify-center transition-all hover:bg-gray-100"
+        >
+          <img src={ArrowLeft} alt="left-arrow" className="h-3" />
+        </button>
+      </div>
+      <div
+        ref={containerRef}
+        className="flex flex-nowrap overflow-x-auto no-scrollbar md:space-x-4 scroll-smooth snap-x"
+      >
+        {tabs.map((tab, index) => (
+          <button
+            key={index}
+            onClick={() => setActiveTab(tab)}
+            className={`snap-start h-9 rounded-3xl px-4 font-bold hover:text-neutral-600 dark:hover:text-white/60 ${
+              activeTab === tab
+                ? 'bg-neutral-100 text-neutral-600  dark:bg-dark-charcoal dark:text-white/60'
+                : 'text-gray-6000'
+            }`}
+          >
+            {tab}
+          </button>
+        ))}
+      </div>
+      <div className="md:hidden z-10">
+        <button
+          onClick={() => scrollTabs(1)}
+          className="flex h-6 w-6 rounded-full items-center justify-center hover:bg-gray-100"
+        >
+          <img src={ArrowRight} alt="right-arrow" className="h-3" />
+        </button>
+      </div>
+    </div>
+  );
+};
+
+export default SettingsBar;