Merge branch 'main' of https://github.com/utin-francis-peter/DocsGPT into feat/sources-in-react-widget

.github/dependabot.yml

@@ -13,3 +13,7 @@ updates:
     directory: "/frontend" # Location of package manifests
     schedule:
       interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/holopin.yml

@@ -1,5 +1,11 @@
-organization: arc53
-defaultSticker: clqmdf0ed34290glbvqh0kzxd
+organization: docsgpt
+defaultSticker: cm1ulwkkl180570cl82rtzympu
 stickers:
-  - id: clqmdf0ed34290glbvqh0kzxd
-    alias: festive
+  - id: cm1ulwkkl180570cl82rtzympu
+    alias: contributor2024
+  - id: cm1ureg8o130450cl8c1po6mil
+    alias: api
+  - id: cm1urhmag148240cl8yvqxkthx
+    alias: lpc
+  - id: cm1urlcpq622090cl2tvu4w71y
+    alias: lexeu

.github/workflows/ci.yml

@@ -12,22 +12,22 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

.github/workflows/cife.yml

@@ -12,22 +12,22 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

.github/workflows/docker-develop-build.yml

@@ -14,22 +14,22 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

.github/workflows/docker-develop-fe-build.yml

@@ -14,22 +14,22 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Lint with Ruff
         uses: chartboost/ruff-action@v1

.github/workflows/pytest.yml

@@ -8,9 +8,9 @@ jobs:
       matrix:
         python-version: ["3.11"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -24,7 +24,7 @@ jobs:
           python -m pytest --cov=application --cov-report=xml
       - name: Upload coverage reports to Codecov
         if: github.event_name == 'pull_request' && matrix.python-version == '3.11'
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/sync_fork.yaml

@@ -17,7 +17,7 @@ jobs:
     steps:
       # Step 1: run a standard checkout action
       - name: Checkout target repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Step 2: run the sync action
       - name: Sync upstream changes

README.md

@@ -35,7 +35,8 @@ We're eager to provide personalized assistance when deploying your DocsGPT to a
 
 [Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
 
-![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
+
+<img src="https://github.com/user-attachments/assets/9a1f21de-7a15-4e42-9424-70d22ba5a913" alt="video-example-of-docs-gpt" width="1000" height="500">
 
 ## Roadmap
 
@@ -202,4 +203,9 @@ We as members, contributors, and leaders, pledge to make participation in our co
 
 The source code license is [MIT](https://opensource.org/license/mit/), as described in the [LICENSE](LICENSE) file.
 
-Built with [:bird: :link: LangChain](https://github.com/hwchase17/langchain)
+<p>This project is supported by:</p>
+<p>
+  <a href="https://www.digitalocean.com/?utm_medium=opensource&utm_source=DocsGPT">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px">
+  </a>
+</p>

application/api/answer/routes.py

@@ -292,6 +292,7 @@ class Stream(Resource):
     def post(self):
         data = request.get_json()
         required_fields = ["question"]
+
         missing_fields = check_required_fields(data, required_fields)
         if missing_fields:
             return missing_fields
@@ -422,7 +423,7 @@ class Answer(Resource):
     @api.doc(description="Provide an answer based on the question and retriever")
     def post(self):
         data = request.get_json()
-        required_fields = ["question"]
+        required_fields = ["question"]       
         missing_fields = check_required_fields(data, required_fields)
         if missing_fields:
             return missing_fields

application/api/user/routes.py

@@ -7,7 +7,7 @@ from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
 from flask import Blueprint, jsonify, make_response, request
-from flask_restx import fields, Namespace, Resource
+from flask_restx import inputs, fields, Namespace, Resource
 from pymongo import MongoClient
 from werkzeug.utils import secure_filename
 
@@ -340,6 +340,8 @@ class UploadFile(Resource):
                         ".epub",
                         ".html",
                         ".mdx",
+                        ".json",
+                        ".xlsx",
                     ],
                     job_name,
                     final_filename,
@@ -802,7 +804,7 @@ class ShareConversation(Resource):
         if missing_fields:
             return missing_fields
 
-        is_promptable = request.args.get("isPromptable")
+        is_promptable = request.args.get("isPromptable", type=inputs.boolean)
         if is_promptable is None:
             return make_response(
                 jsonify({"success": False, "message": "isPromptable is required"}), 400
@@ -831,7 +833,7 @@ class ShareConversation(Resource):
                 uuid.uuid4(), UuidRepresentation.STANDARD
             )
 
-            if is_promptable.lower() == "true":
+            if is_promptable:
                 prompt_id = data.get("prompt_id", "default")
                 chunks = data.get("chunks", "2")
 
@@ -859,7 +861,7 @@ class ShareConversation(Resource):
                             "conversation_id": DBRef(
                                 "conversations", ObjectId(conversation_id)
                             ),
-                            "isPromptable": is_promptable.lower() == "true",
+                            "isPromptable": is_promptable,
                             "first_n_queries": current_n_queries,
                             "user": user,
                             "api_key": api_uuid,
@@ -883,7 +885,7 @@ class ShareConversation(Resource):
                                     "$ref": "conversations",
                                     "$id": ObjectId(conversation_id),
                                 },
-                                "isPromptable": is_promptable.lower() == "true",
+                                "isPromptable": is_promptable,
                                 "first_n_queries": current_n_queries,
                                 "user": user,
                                 "api_key": api_uuid,
@@ -918,7 +920,7 @@ class ShareConversation(Resource):
                                 "$ref": "conversations",
                                 "$id": ObjectId(conversation_id),
                             },
-                            "isPromptable": is_promptable.lower() == "true",
+                            "isPromptable": is_promptable,
                             "first_n_queries": current_n_queries,
                             "user": user,
                             "api_key": api_uuid,
@@ -939,7 +941,7 @@ class ShareConversation(Resource):
                     "conversation_id": DBRef(
                         "conversations", ObjectId(conversation_id)
                     ),
-                    "isPromptable": is_promptable.lower() == "false",
+                    "isPromptable": is_promptable,
                     "first_n_queries": current_n_queries,
                     "user": user,
                 }
@@ -962,7 +964,7 @@ class ShareConversation(Resource):
                             "$ref": "conversations",
                             "$id": ObjectId(conversation_id),
                         },
-                        "isPromptable": is_promptable.lower() == "false",
+                        "isPromptable": is_promptable,
                         "first_n_queries": current_n_queries,
                         "user": user,
                     }

application/cache.py

@@ -0,0 +1,93 @@
+import redis
+import time
+import json
+import logging
+from threading import Lock
+from application.core.settings import settings
+from application.utils import get_hash
+
+logger = logging.getLogger(__name__)
+
+_redis_instance = None
+_instance_lock = Lock()
+
+def get_redis_instance():
+    global _redis_instance
+    if _redis_instance is None:
+        with _instance_lock:
+            if _redis_instance is None:
+                try:
+                    _redis_instance = redis.Redis.from_url(settings.CACHE_REDIS_URL, socket_connect_timeout=2)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+                    _redis_instance = None
+    return _redis_instance
+
+def gen_cache_key(*messages, model="docgpt"):
+    if not all(isinstance(msg, dict) for msg in messages):
+        raise ValueError("All messages must be dictionaries.")
+    messages_str = json.dumps(list(messages), sort_keys=True)
+    combined = f"{model}_{messages_str}"
+    cache_key = get_hash(combined)
+    return cache_key
+
+def gen_cache(func):
+    def wrapper(self, model, messages, *args, **kwargs):
+        try:
+            cache_key = gen_cache_key(*messages)
+            redis_client = get_redis_instance()
+            if redis_client:
+                try:
+                    cached_response = redis_client.get(cache_key)
+                    if cached_response:
+                        return cached_response.decode('utf-8')
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            result = func(self, model, messages, *args, **kwargs)
+            if redis_client:
+                try:
+                    redis_client.set(cache_key, result, ex=1800)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            return result
+        except ValueError as e:
+            logger.error(e)
+            return "Error: No user message found in the conversation to generate a cache key."
+    return wrapper
+
+def stream_cache(func):
+    def wrapper(self, model, messages, stream, *args, **kwargs):
+        cache_key = gen_cache_key(*messages)
+        logger.info(f"Stream cache key: {cache_key}")
+        
+        redis_client = get_redis_instance()
+        if redis_client:
+            try:
+                cached_response = redis_client.get(cache_key)
+                if cached_response:
+                    logger.info(f"Cache hit for stream key: {cache_key}")
+                    cached_response = json.loads(cached_response.decode('utf-8'))
+                    for chunk in cached_response:
+                        yield chunk
+                        time.sleep(0.03)
+                    return
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+
+        result = func(self, model, messages, stream, *args, **kwargs)
+        stream_cache_data = []
+        
+        for chunk in result:
+            stream_cache_data.append(chunk)
+            yield chunk
+        
+        if redis_client:
+            try:
+                redis_client.set(cache_key, json.dumps(stream_cache_data), ex=1800)
+                logger.info(f"Stream cache saved for key: {cache_key}")
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+        
+    return wrapper

application/core/settings.py

@@ -18,9 +18,12 @@ class Settings(BaseSettings):
     DEFAULT_MAX_HISTORY: int = 150
     MODEL_TOKEN_LIMITS: dict = {"gpt-3.5-turbo": 4096, "claude-2": 1e5}
     UPLOAD_FOLDER: str = "inputs"
-    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant" or "milvus"
+    VECTOR_STORE: str = "faiss" #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb"
     RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
 
+    # LLM Cache
+    CACHE_REDIS_URL: str = "redis://localhost:6379/2"
+
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
 
     API_KEY: Optional[str] = None  # LLM api key
@@ -67,6 +70,9 @@ class Settings(BaseSettings):
     MILVUS_URI: Optional[str] = "./milvus_local.db"   # milvus lite version as default
     MILVUS_TOKEN: Optional[str] = ""
 
+    # LanceDB vectorstore config
+    LANCEDB_PATH: str = "/tmp/lancedb"  # Path where LanceDB stores its local data
+    LANCEDB_TABLE_NAME: Optional[str] = "docsgpts"  # Name of the table to use for storing vectors
     BRAVE_SEARCH_API_KEY: Optional[str] = None
 
     FLASK_DEBUG_MODE: bool = False

application/llm/base.py

@@ -1,28 +1,29 @@
 from abc import ABC, abstractmethod
 from application.usage import gen_token_usage, stream_token_usage
+from application.cache import stream_cache, gen_cache
 
 
 class BaseLLM(ABC):
     def __init__(self):
         self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
 
-    def _apply_decorator(self, method, decorator, *args, **kwargs):
-        return decorator(method, *args, **kwargs)
+    def _apply_decorator(self, method, decorators, *args, **kwargs):
+        for decorator in decorators:
+            method = decorator(method)
+        return method(self, *args, **kwargs)
 
     @abstractmethod
     def _raw_gen(self, model, messages, stream, *args, **kwargs):
         pass
 
     def gen(self, model, messages, stream=False, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen, gen_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
-        )
+        decorators = [gen_token_usage, gen_cache]
+        return self._apply_decorator(self._raw_gen, decorators=decorators, model=model, messages=messages, stream=stream, *args, **kwargs)
 
     @abstractmethod
     def _raw_gen_stream(self, model, messages, stream, *args, **kwargs):
         pass
 
     def gen_stream(self, model, messages, stream=True, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen_stream, stream_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
-        )
+        decorators = [stream_cache, stream_token_usage]
+        return self._apply_decorator(self._raw_gen_stream, decorators=decorators, model=model, messages=messages, stream=stream, *args, **kwargs)

application/parser/file/bulk.py

@@ -11,6 +11,7 @@ from application.parser.file.html_parser import HTMLParser
 from application.parser.file.markdown_parser import MarkdownParser
 from application.parser.file.rst_parser import RstParser
 from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser
+from application.parser.file.json_parser import JSONParser
 from application.parser.schema.base import Document
 
 DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
@@ -23,6 +24,7 @@ DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
     ".rst": RstParser(),
     ".html": HTMLParser(),
     ".mdx": MarkdownParser(),
+    ".json":JSONParser(),
 }
 
 

application/parser/file/json_parser.py

@@ -0,0 +1,57 @@
+import json
+from typing import Any, Dict, List, Union
+from pathlib import Path
+
+from application.parser.file.base_parser import BaseParser
+
+class JSONParser(BaseParser):
+    r"""JSON (.json) parser.
+
+    Parses JSON files into a list of strings or a concatenated document.
+    It handles both JSON objects (dictionaries) and arrays (lists).
+
+    Args:
+        concat_rows (bool): Whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each item in the JSON.
+            True by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        json_config (dict): Options for parsing JSON. Can be used to specify options like
+        custom decoding or formatting. Set to empty dict by default.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            row_joiner: str = "\n",
+            json_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._row_joiner = row_joiner
+        self._json_config = json_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse JSON file."""
+        
+        with open(file, 'r', encoding='utf-8') as f:
+                data = json.load(f, **self._json_config)
+
+        if isinstance(data, dict):
+            data = [data]
+
+        if self._concat_rows:
+            return self._row_joiner.join([str(item) for item in data])
+        else:
+            return data

application/parser/remote/github_loader.py

@@ -3,6 +3,7 @@ import requests
 from typing import List
 from application.parser.remote.base import BaseRemote
 from langchain_core.documents import Document
+import mimetypes
 
 class GitHubLoader(BaseRemote):
     def __init__(self):
@@ -18,13 +19,17 @@ class GitHubLoader(BaseRemote):
 
         if response.status_code == 200:
             content = response.json()
+            mime_type, _ = mimetypes.guess_type(file_path)  # Guess the MIME type based on the file extension
+
             if content.get("encoding") == "base64":
-                try:
-                    decoded_content = base64.b64decode(content["content"]).decode("utf-8")
-                    return f"Filename: {file_path}\n\n{decoded_content}"
-                except Exception as e:
-                    print(f"Error decoding content for {file_path}: {e}")
-                    raise
+                if mime_type and mime_type.startswith("text"):  # Handle only text files
+                    try:
+                        decoded_content = base64.b64decode(content["content"]).decode("utf-8")
+                        return f"Filename: {file_path}\n\n{decoded_content}"
+                    except Exception as e:
+                        raise e
+                else:
+                    return f"Filename: {file_path} is a binary file and was skipped."
             else:
                 return f"Filename: {file_path}\n\n{content['content']}"
         else:

application/requirements.txt

@@ -4,7 +4,7 @@ beautifulsoup4==4.12.3
 celery==5.3.6
 dataclasses-json==0.6.7
 docx2txt==0.8
-duckduckgo-search==6.2.6
+duckduckgo-search==6.3.0
 ebooklib==0.18
 elastic-transport==8.15.0
 elasticsearch==8.15.1
@@ -54,7 +54,7 @@ pathable==0.4.3
 pillow==10.4.0
 portalocker==2.10.1
 prance==23.6.21.0
-primp==0.6.2
+primp==0.6.3
 prompt-toolkit==3.0.47
 protobuf==5.28.2
 py==1.11.0

application/utils.py

@@ -1,6 +1,8 @@
 import tiktoken
+import hashlib
 from flask import jsonify, make_response
 
+
 _encoding = None
 
 
@@ -39,3 +41,8 @@ def check_required_fields(data, required_fields):
             400,
         )
     return None
+
+
+def get_hash(data):
+    return hashlib.md5(data.encode()).hexdigest()
+

application/vectorstore/lancedb.py

@@ -0,0 +1,119 @@
+from typing import List, Optional
+import importlib
+from application.vectorstore.base import BaseVectorStore
+from application.core.settings import settings
+
+class LanceDBVectorStore(BaseVectorStore):
+    """Class for LanceDB Vector Store integration."""
+
+    def __init__(self, path: str = settings.LANCEDB_PATH,
+                 table_name_prefix: str = settings.LANCEDB_TABLE_NAME,
+                 source_id: str = None,
+                 embeddings_key: str = "embeddings"):
+        """Initialize the LanceDB vector store."""
+        super().__init__()
+        self.path = path
+        self.table_name = f"{table_name_prefix}_{source_id}" if source_id else table_name_prefix
+        self.embeddings_key = embeddings_key
+        self._lance_db = None
+        self.docsearch = None
+        self._pa = None  # PyArrow (pa) will be lazy loaded
+
+    @property
+    def pa(self):
+        """Lazy load pyarrow module."""
+        if self._pa is None:
+            self._pa = importlib.import_module("pyarrow")
+        return self._pa
+
+    @property
+    def lancedb(self):
+        """Lazy load lancedb module."""
+        if not hasattr(self, "_lancedb_module"):
+            self._lancedb_module = importlib.import_module("lancedb")
+        return self._lancedb_module
+
+    @property
+    def lance_db(self):
+        """Lazy load the LanceDB connection."""
+        if self._lance_db is None:
+            self._lance_db = self.lancedb.connect(self.path)
+        return self._lance_db
+
+    @property
+    def table(self):
+        """Lazy load the LanceDB table."""
+        if self.docsearch is None:
+            if self.table_name in self.lance_db.table_names():
+                self.docsearch = self.lance_db.open_table(self.table_name)
+            else:
+                self.docsearch = None
+        return self.docsearch
+
+    def ensure_table_exists(self):
+        """Ensure the table exists before performing operations."""
+        if self.table is None:
+            embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
+            schema = self.pa.schema([
+                self.pa.field("vector", self.pa.list_(self.pa.float32(), list_size=embeddings.dimension)),
+                self.pa.field("text", self.pa.string()),
+                self.pa.field("metadata", self.pa.struct([
+                    self.pa.field("key", self.pa.string()),
+                    self.pa.field("value", self.pa.string())
+                ]))
+            ])
+            self.docsearch = self.lance_db.create_table(self.table_name, schema=schema)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]] = None, source_id: str = None):
+        """Add texts with metadata and their embeddings to the LanceDB table."""
+        embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_documents(texts)
+        vectors = []
+        for embedding, text, metadata in zip(embeddings, texts, metadatas or [{}] * len(texts)):
+            if source_id:
+                metadata["source_id"] = source_id
+            metadata_struct = [{"key": k, "value": str(v)} for k, v in metadata.items()]
+            vectors.append({
+                "vector": embedding,
+                "text": text,
+                "metadata": metadata_struct
+            })
+        self.ensure_table_exists()
+        self.docsearch.add(vectors)
+
+    def search(self, query: str, k: int = 2, *args, **kwargs):
+        """Search LanceDB for the top k most similar vectors."""
+        self.ensure_table_exists()
+        query_embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_query(query)
+        results = self.docsearch.search(query_embedding).limit(k).to_list()
+        return [(result["_distance"], result["text"], result["metadata"]) for result in results]
+
+    def delete_index(self):
+        """Delete the entire LanceDB index (table)."""
+        if self.table:
+            self.lance_db.drop_table(self.table_name)
+
+    def assert_embedding_dimensions(self, embeddings):
+        """Ensure that embedding dimensions match the table index dimensions."""
+        word_embedding_dimension = embeddings.dimension
+        if self.table:
+            table_index_dimension = len(self.docsearch.schema["vector"].type.value_type)
+            if word_embedding_dimension != table_index_dimension:
+                raise ValueError(
+                    f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) "
+                    f"!= table index dimension ({table_index_dimension})"
+                )
+
+    def filter_documents(self, filter_condition: dict) -> List[dict]:
+        """Filter documents based on certain conditions."""
+        self.ensure_table_exists()
+
+        # Ensure source_id exists in the filter condition
+        if 'source_id' not in filter_condition:
+            raise ValueError("filter_condition must contain 'source_id'")
+
+        source_id = filter_condition["source_id"]
+
+        # Use LanceDB's native filtering if supported, otherwise filter manually
+        filtered_data = self.docsearch.filter(lambda x: x.metadata and x.metadata.get("source_id") == source_id).to_list()
+
+        return filtered_data

docker-compose.yaml

@@ -20,6 +20,7 @@ services:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - CACHE_REDIS_URL=redis://redis:6379/2
     ports:
       - "7091:7091"
     volumes:
@@ -41,6 +42,7 @@ services:
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
+      - CACHE_REDIS_URL=redis://redis:6379/2
     depends_on:
       - redis
       - mongo

docs/package-lock.json

@@ -7,7 +7,7 @@
       "license": "MIT",
       "dependencies": {
         "@vercel/analytics": "^1.1.1",
-        "docsgpt": "^0.4.1",
+        "docsgpt": "^0.4.3",
         "next": "^14.2.12",
         "nextra": "^2.13.2",
         "nextra-theme-docs": "^2.13.2",
@@ -422,11 +422,6 @@
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@bpmn-io/snarkdown": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/@bpmn-io/snarkdown/-/snarkdown-2.2.0.tgz",
-      "integrity": "sha512-bVD7FIoaBDZeCJkMRgnBPDeptPlto87wt2qaCjf5t8iLaevDmTPaREd6FpBEGsHlUdHFFZWRk4qAoEC5So2M0Q=="
-    },
     "node_modules/@braintree/sanitize-url": {
       "version": "6.0.4",
       "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-6.0.4.tgz",
@@ -3162,30 +3157,6 @@
         "cytoscape": "^3.2.0"
       }
     },
-    "node_modules/cytoscape-fcose": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz",
-      "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==",
-      "dependencies": {
-        "cose-base": "^2.2.0"
-      },
-      "peerDependencies": {
-        "cytoscape": "^3.2.0"
-      }
-    },
-    "node_modules/cytoscape-fcose/node_modules/cose-base": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz",
-      "integrity": "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==",
-      "dependencies": {
-        "layout-base": "^2.0.0"
-      }
-    },
-    "node_modules/cytoscape-fcose/node_modules/layout-base": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz",
-      "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg=="
-    },
     "node_modules/d3": {
       "version": "7.8.5",
       "resolved": "https://registry.npmjs.org/d3/-/d3-7.8.5.tgz",
@@ -3697,13 +3668,12 @@
       }
     },
     "node_modules/docsgpt": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/docsgpt/-/docsgpt-0.4.1.tgz",
-      "integrity": "sha512-9oH638vIg8I+zsjLV5Rp21yYniAtiTcyuBSByqWl2KoBdF/8vDSmr491l8n+ikbaTLiCW4uRU0p0r3BvRizy2Q==",
+      "version": "0.4.3",
+      "resolved": "https://registry.npmjs.org/docsgpt/-/docsgpt-0.4.3.tgz",
+      "integrity": "sha512-svLM6xEg4iUtb7HuR1qwC95K4ctvTky8gXRXgqtDIUC5Fg4zeHwivbmaFkBbP3N+bcqWVWCJK9DfJfW+OjTeuA==",
       "license": "Apache-2.0",
       "dependencies": {
         "@babel/plugin-transform-flow-strip-types": "^7.23.3",
-        "@bpmn-io/snarkdown": "^2.2.0",
         "@parcel/resolver-glob": "^2.12.0",
         "@parcel/transformer-svg-react": "^2.12.0",
         "@parcel/transformer-typescript-tsc": "^2.12.0",
@@ -3715,6 +3685,7 @@
         "flow-bin": "^0.229.2",
         "i": "^0.3.7",
         "install": "^0.13.0",
+        "markdown-it": "^14.1.0",
         "npm": "^10.5.0",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
@@ -3807,9 +3778,9 @@
       "integrity": "sha512-/if4Ueg0GUQlhCrW2ZlXwDAm40ipuKo+OgeHInlL8sbjt+hzISxZK949fZeJaVsheamrzANXvw1zQTvbxTvSHw=="
     },
     "node_modules/elkjs": {
-      "version": "0.8.2",
-      "resolved": "https://registry.npmjs.org/elkjs/-/elkjs-0.8.2.tgz",
-      "integrity": "sha512-L6uRgvZTH+4OF5NE/MBbzQx/WYpru1xCBE9respNj6qznEewGUIfhzmm7horWWxbNO2M0WckQypGctR8lH79xQ=="
+      "version": "0.9.3",
+      "resolved": "https://registry.npmjs.org/elkjs/-/elkjs-0.9.3.tgz",
+      "integrity": "sha512-f/ZeWvW/BCXbhGEf1Ujp29EASo/lk1FDnETgNKwJrsVvGZhUWCZyg3xLJjAsxfOmt8KjswHmI5EwCQcPMpOYhQ=="
     },
     "node_modules/entities": {
       "version": "4.5.0",
@@ -4859,6 +4830,15 @@
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
       "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
     },
+    "node_modules/linkify-it": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz",
+      "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==",
+      "license": "MIT",
+      "dependencies": {
+        "uc.micro": "^2.0.0"
+      }
+    },
     "node_modules/lmdb": {
       "version": "2.8.5",
       "resolved": "https://registry.npmjs.org/lmdb/-/lmdb-2.8.5.tgz",
@@ -4944,6 +4924,29 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/markdown-it": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz",
+      "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==",
+      "license": "MIT",
+      "dependencies": {
+        "argparse": "^2.0.1",
+        "entities": "^4.4.0",
+        "linkify-it": "^5.0.0",
+        "mdurl": "^2.0.0",
+        "punycode.js": "^2.3.1",
+        "uc.micro": "^2.1.0"
+      },
+      "bin": {
+        "markdown-it": "bin/markdown-it.mjs"
+      }
+    },
+    "node_modules/markdown-it/node_modules/argparse": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
+      "license": "Python-2.0"
+    },
     "node_modules/markdown-table": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.3.tgz",
@@ -5487,23 +5490,29 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/mdurl": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
+      "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==",
+      "license": "MIT"
+    },
     "node_modules/mermaid": {
-      "version": "10.6.1",
-      "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.6.1.tgz",
-      "integrity": "sha512-Hky0/RpOw/1il9X8AvzOEChfJtVvmXm+y7JML5C//ePYMy0/9jCEmW1E1g86x9oDfW9+iVEdTV/i+M6KWRNs4A==",
+      "version": "10.9.3",
+      "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.9.3.tgz",
+      "integrity": "sha512-V80X1isSEvAewIL3xhmz/rVmc27CVljcsbWxkxlWJWY/1kQa4XOABqpDl2qQLGKzpKm6WbTfUEKImBlUfFYArw==",
       "dependencies": {
         "@braintree/sanitize-url": "^6.0.1",
         "@types/d3-scale": "^4.0.3",
         "@types/d3-scale-chromatic": "^3.0.0",
-        "cytoscape": "^3.23.0",
+        "cytoscape": "^3.28.1",
         "cytoscape-cose-bilkent": "^4.1.0",
-        "cytoscape-fcose": "^2.1.0",
         "d3": "^7.4.0",
         "d3-sankey": "^0.12.3",
         "dagre-d3-es": "7.0.10",
         "dayjs": "^1.11.7",
-        "dompurify": "^3.0.5",
-        "elkjs": "^0.8.2",
+        "dompurify": "^3.0.5 <3.1.7",
+        "elkjs": "^0.9.0",
+        "katex": "^0.16.9",
         "khroma": "^2.0.0",
         "lodash-es": "^4.17.21",
         "mdast-util-from-markdown": "^1.3.0",
@@ -9338,6 +9347,15 @@
       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
       "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ=="
     },
+    "node_modules/punycode.js": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",
+      "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/react": {
       "version": "18.2.0",
       "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
@@ -10052,9 +10070,9 @@
       }
     },
     "node_modules/typescript": {
-      "version": "5.6.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz",
-      "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==",
+      "version": "5.6.3",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz",
+      "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==",
       "peer": true,
       "bin": {
         "tsc": "bin/tsc",
@@ -10064,6 +10082,12 @@
         "node": ">=14.17"
       }
     },
+    "node_modules/uc.micro": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz",
+      "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
+      "license": "MIT"
+    },
     "node_modules/unified": {
       "version": "10.1.2",
       "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.2.tgz",

docs/package.json

@@ -7,7 +7,7 @@
   "license": "MIT",
   "dependencies": {
     "@vercel/analytics": "^1.1.1",
-    "docsgpt": "^0.4.1",
+    "docsgpt": "^0.4.3",
     "next": "^14.2.12",
     "nextra": "^2.13.2",
     "nextra-theme-docs": "^2.13.2",

extensions/react-widget/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docsgpt",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "private": false,
   "description": "DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.",
   "source": "./src/index.html",

extensions/react-widget/src/components/DocsGPTWidget.tsx

@@ -458,7 +458,7 @@ export const DocsGPTWidget = ({
 
             }
             else {
-              const result = data.answer;
+              const result = data.answer ? data.answer : ''; //Fallback to an empty string if data.answer is undefined
               const streamingResponse = queries[queries.length - 1].response ? queries[queries.length - 1].response : '';
               const updatedQueries = [...queries];
               updatedQueries[updatedQueries.length - 1].response = streamingResponse + result;

extensions/slack-bot/.gitignore

@@ -0,0 +1,3 @@
+.env
+.venv/
+get-pip.py

extensions/slack-bot/Readme.md

@@ -0,0 +1,84 @@
+
+# Slack Bot Configuration Guide
+
+> **Note:** The following guidelines must be followed on the [Slack API website](https://api.slack.com/) for setting up your Slack app and generating the necessary tokens.
+
+## Step-by-Step Instructions
+
+### 1. Navigate to Your Apps
+- Go to the Slack API page for apps and select **Create an App** from the “From Scratch” option.
+
+### 2. App Creation
+- Name your app and choose the workspace where you wish to add the assistant. 
+
+### 3. Enabling Socket Mode
+- Navigate to **Settings > Socket Mode** and enable **Socket Mode**. 
+- This action will generate an App-level token. Select the `connections:write` scope and copy the App-level token for future use.
+
+### 4. Socket Naming
+- Assign a name to your socket as per your preference.
+
+### 5. Basic Information Setup
+- Go to **Basic Information** (under **Settings**) and configure the following:
+  - Assistant name
+  - App icon
+  - Background color 
+
+### 6. Bot Token and Permissions
+- In the **OAuth & Permissions** option found under the **Features** section, retrieve the Bot Token. Save it for future usage.
+- You will also need to add specific bot token scopes:
+  - `app_mentions:read`
+  - `assistant:write`
+  - `chat:write`
+  - `chat:write.public`
+  - `im:history`
+
+### 7. Enable Events
+- From **Event Subscriptions**, enable events and add the following Bot User events:
+  - `app_mention`
+  - `assistant_thread_context_changed`
+  - `assistant_thread_started`
+  - `message.im`
+
+### 8. Agent/Assistant Toggle
+- In the **Features > Agent & Assistants** section, toggle on the Agent or Assistant option. 
+- In the **Suggested Prompts** setting, leave it as `dynamic` (this is the default setting).
+
+---
+
+## Code-Side Configuration Guide
+
+This section focuses on generating and setting up the necessary tokens in the `.env` file, using the `.env-example` as a template.
+
+### Step 1: Generating Required Keys
+
+1. **SLACK_APP_TOKEN**
+   - Navigate to **Settings > Socket Mode** in the Slack API and enable **Socket Mode**.
+   - Copy the App-level token generated (usually starts with `xapp-`).
+
+2. **SLACK_BOT_TOKEN**
+   - Go to **OAuth & Permissions** (under the **Features** section in Slack API).
+   - Retrieve the **Bot Token** (starts with `xoxb-`).
+
+3. **DOCSGPT_API_KEY**
+   - Go to the **DocsGPT website**.
+   - Navigate to **Settings > Chatbots > Create New** to generate a DocsGPT API Key.
+   - Copy the generated key for use.
+
+### Step 2: Creating and Updating the `.env` File
+
+1. Create a new `.env` file in the root of your project (if it doesn’t already exist).
+2. Use the `.env-example` as a reference and update the file with the following keys and values:
+
+```bash
+# .env file
+SLACK_APP_TOKEN=xapp-your-generated-app-token
+SLACK_BOT_TOKEN=xoxb-your-generated-bot-token
+DOCSGPT_API_KEY=your-docsgpt-generated-api-key
+```
+
+Replace the placeholder values with the actual tokens generated from the Slack API and DocsGPT as per the steps outlined above.
+
+---
+
+This concludes the guide for both setting up the Slack API and configuring the `.env` file on the code side.

extensions/slack-bot/app.py

@@ -0,0 +1,112 @@
+import os
+import hashlib
+import httpx
+import re
+from slack_bolt.async_app import AsyncApp
+from slack_bolt.adapter.socket_mode.async_handler import AsyncSocketModeHandler
+from dotenv import load_dotenv
+
+load_dotenv()
+API_BASE = os.getenv("API_BASE", "https://gptcloud.arc53.com")
+API_URL =  API_BASE + "/api/answer"
+
+# Slack bot token and signing secret
+SLACK_BOT_TOKEN = os.getenv("SLACK_BOT_TOKEN")
+SLACK_APP_TOKEN = os.getenv("SLACK_APP_TOKEN")
+
+# OpenAI API key for DocsGPT (replace this with your actual API key)
+DOCSGPT_API_KEY = os.getenv("DOCSGPT_API_KEY")
+
+# Initialize Slack app
+app = AsyncApp(token=SLACK_BOT_TOKEN)
+
+def encode_conversation_id(conversation_id: str) -> str:
+    """
+        Encodes 11 length Slack conversation_id to 12 length string
+        Args:
+        conversation_id (str): The 11 digit slack conversation_id.
+        Returns:
+            str: Hashed id.
+    """    
+    # Create a SHA-256 hash of the string
+    hashed_id = hashlib.sha256(conversation_id.encode()).hexdigest()
+
+    # Take the first 24 characters of the hash
+    hashed_24_char_id = hashed_id[:24]
+    return hashed_24_char_id
+
+async def generate_answer(question: str, messages: list, conversation_id: str | None) -> dict:
+    """Generates an answer using the external API."""
+    payload = {
+        "question": question,
+        "api_key": DOCSGPT_API_KEY,
+        "history": messages,
+        "conversation_id": conversation_id,
+    }
+    headers = {
+        "Content-Type": "application/json; charset=utf-8"
+    }
+    timeout = 60.0
+    async with httpx.AsyncClient() as client:
+        response = await client.post(API_URL, json=payload, headers=headers, timeout=timeout)
+
+        if response.status_code == 200:
+            data = response.json()
+            conversation_id = data.get("conversation_id")
+            answer = data.get("answer", "Sorry, I couldn't find an answer.")
+            return {"answer": answer, "conversation_id": conversation_id}
+        else:
+            print(response.json())
+            return {"answer": "Sorry, I couldn't find an answer.", "conversation_id": None}
+
+@app.message(".*")
+async def message_docs(message, say):
+    client = app.client
+    channel = message['channel']
+    thread_ts = message['thread_ts']
+    user_query = message['text']    
+    await client.assistant_threads_setStatus(
+        channel_id = channel,
+        thread_ts = thread_ts,
+        status = "is generating your answer...",
+    )
+
+    docs_gpt_channel_id = encode_conversation_id(thread_ts)
+    
+    # Get response from DocsGPT
+    response = await generate_answer(user_query,[], docs_gpt_channel_id)
+    answer = convert_to_slack_markdown(response['answer'])
+
+    # Respond in Slack
+    await client.chat_postMessage(text = answer, mrkdwn= True, channel= message['channel'],
+        thread_ts = message['thread_ts'],)
+
+def convert_to_slack_markdown(markdown_text: str):
+    # Convert bold **text** to *text* for Slack
+    slack_text = re.sub(r'\*\*(.*?)\*\*', r'*\1*', markdown_text)  # **text** to *text*
+
+    # Convert italics _text_ to _text_ for Slack
+    slack_text = re.sub(r'_(.*?)_', r'_\1_', slack_text)  # _text_ to _text_
+
+    # Convert inline code `code` to `code` (Slack supports backticks for inline code)
+    slack_text = re.sub(r'`(.*?)`', r'`\1`', slack_text)
+
+    # Convert bullet points with single or no spaces to filled bullets (•)
+    slack_text = re.sub(r'^\s{0,1}[-*]\s+', ' • ', slack_text, flags=re.MULTILINE)
+
+    # Convert bullet points with multiple spaces to hollow bullets (◦)
+    slack_text = re.sub(r'^\s{2,}[-*]\s+', '\t◦ ', slack_text, flags=re.MULTILINE)
+
+    # Convert headers (##) to bold in Slack
+    slack_text = re.sub(r'^\s*#{1,6}\s*(.*?)$', r'*\1*', slack_text, flags=re.MULTILINE)
+
+    return slack_text
+
+async def main():
+    handler = AsyncSocketModeHandler(app, os.environ["SLACK_APP_TOKEN"])
+    await handler.start_async()
+
+# Start the app
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())

extensions/slack-bot/requirements.txt

@@ -0,0 +1,10 @@
+aiohttp>=3,<4
+certifi==2024.7.4
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+idna==3.7
+python-dotenv==1.0.1
+sniffio==1.3.1
+slack-bolt==1.21.0
+bson==0.5.10

frontend/src/App.tsx

@@ -19,7 +19,7 @@ function MainLayout() {
     <div className="dark:bg-raisin-black relative h-screen overflow-auto">
       <Navigation navOpen={navOpen} setNavOpen={setNavOpen} />
       <div
-        className={`h-[calc(100dvh-64px)] sm:h-screen ${
+        className={`h-[calc(100dvh-64px)] md:h-screen ${
           !isMobile
             ? `ml-0 ${!navOpen ? 'md:mx-auto lg:mx-auto' : 'md:ml-72'}`
             : 'ml-0 md:ml-16'
@@ -32,9 +32,9 @@ function MainLayout() {
 }
 
 export default function App() {
-  const [,,componentMounted] = useDarkTheme();
-  if(!componentMounted) {
-    return <div />
+  const [, , componentMounted] = useDarkTheme();
+  if (!componentMounted) {
+    return <div />;
   }
   return (
     <div className="h-full relative overflow-auto">

frontend/src/Navigation.tsx

@@ -2,16 +2,15 @@ import { useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
 import { NavLink, useNavigate } from 'react-router-dom';
-
 import conversationService from './api/services/conversationService';
 import userService from './api/services/userService';
 import Add from './assets/add.svg';
+import openNewChat from './assets/openNewChat.svg';
+import Hamburger from './assets/hamburger.svg';
 import DocsGPT3 from './assets/cute_docsgpt3.svg';
 import Discord from './assets/discord.svg';
 import Expand from './assets/expand.svg';
 import Github from './assets/github.svg';
-import Hamburger from './assets/hamburger.svg';
-import Info from './assets/info.svg';
 import SettingGear from './assets/settingGear.svg';
 import Twitter from './assets/TwitterX.svg';
 import UploadIcon from './assets/upload.svg';
@@ -40,7 +39,9 @@ import {
   setSelectedDocs,
   setSourceDocs,
 } from './preferences/preferenceSlice';
+import { selectQueries } from './conversation/conversationSlice';
 import Upload from './upload/Upload';
+import Help from './components/Help';
 
 interface NavigationProps {
   navOpen: boolean;
@@ -63,6 +64,7 @@ NavImage.propTypes = {
 }; */
 export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   const dispatch = useDispatch();
+  const queries = useSelector(selectQueries);
   const docs = useSelector(selectSourceDocs);
   const selectedDocs = useSelector(selectSelectedDocs);
   const conversations = useSelector(selectConversations);
@@ -73,7 +75,6 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   const [isDarkTheme] = useDarkTheme();
   const [isDocsListOpen, setIsDocsListOpen] = useState(false);
   const { t } = useTranslation();
-
   const isApiKeySet = useSelector(selectApiKeyStatus);
   const [apiKeyModalState, setApiKeyModalState] =
     useState<ActiveState>('INACTIVE');
@@ -93,6 +94,9 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
     if (!conversations) {
       fetchConversations();
     }
+    if (queries.length === 0) {
+      resetConversation();
+    }
   }, [conversations, dispatch]);
 
   async function fetchConversations() {
@@ -164,7 +168,11 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
       }),
     );
   };
-
+  const newChat = () => {
+    if (queries && queries?.length > 0) {
+      resetConversation();
+    }
+  };
   async function updateConversationName(updatedConversation: {
     name: string;
     id: string;
@@ -200,26 +208,45 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   return (
     <>
       {!navOpen && (
-        <button
-          className="duration-25 absolute  top-3 left-3 z-20 hidden transition-all md:block"
-          onClick={() => {
-            setNavOpen(!navOpen);
-          }}
-        >
-          <img
-            src={Expand}
-            alt="menu toggle"
-            className={`${
-              !navOpen ? 'rotate-180' : 'rotate-0'
-            } m-auto transition-all duration-200`}
-          />
-        </button>
+        <div className="duration-25 absolute  top-3 left-3 z-20 hidden transition-all md:block">
+          <div className="flex gap-3 items-center">
+            <button
+              onClick={() => {
+                setNavOpen(!navOpen);
+              }}
+            >
+              <img
+                src={Expand}
+                alt="menu toggle"
+                className={`${
+                  !navOpen ? 'rotate-180' : 'rotate-0'
+                } m-auto transition-all duration-200`}
+              />
+            </button>
+            {queries?.length > 0 && (
+              <button
+                onClick={() => {
+                  newChat();
+                }}
+              >
+                <img
+                  src={openNewChat}
+                  alt="open new chat icon"
+                  className="cursor-pointer"
+                />
+              </button>
+            )}
+            <div className="text-[#949494] font-medium text-[20px]">
+              DocsGPT
+            </div>
+          </div>
+        </div>
       )}
       <div
         ref={navRef}
         className={`${
           !navOpen && '-ml-96 md:-ml-[18rem]'
-        } duration-20 fixed top-0 z-20 flex h-full w-72 flex-col border-r-[1px] border-b-0 bg-white transition-all dark:border-r-purple-taupe dark:bg-chinese-black dark:text-white`}
+        } duration-20 fixed top-0 z-40 flex h-full w-72 flex-col border-r-[1px] border-b-0 bg-white transition-all dark:border-r-purple-taupe dark:bg-chinese-black dark:text-white`}
       >
         <div
           className={'visible mt-2 flex h-[6vh] w-full justify-between md:h-12'}
@@ -275,7 +302,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             {t('newChat')}
           </p>
         </NavLink>
-        <div className="mb-auto h-[78vh] overflow-y-auto overflow-x-hidden dark:text-white">
+        <div
+          id="conversationsMainDiv"
+          className="mb-auto h-[78vh] overflow-y-auto overflow-x-hidden dark:text-white"
+        >
           {conversations && conversations.length > 0 ? (
             <div>
               <div className=" my-auto mx-4 mt-2 flex h-6 items-center justify-between gap-4 rounded-3xl">
@@ -304,7 +334,6 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             <></>
           )}
         </div>
-
         <div className="flex h-auto flex-col justify-end text-eerie-black dark:text-white">
           <div className="flex flex-col-reverse border-b-[1px] dark:border-b-purple-taupe">
             <div className="relative my-4 mx-4 flex gap-2">
@@ -359,83 +388,69 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
               </p>
             </NavLink>
           </div>
-          <div className="flex justify-between gap-2 border-b-[1.5px] py-2 dark:border-b-purple-taupe">
-            <NavLink
-              onClick={() => {
-                if (isMobile) {
-                  setNavOpen(!navOpen);
-                }
-                resetConversation();
-              }}
-              to="/about"
-              className={({ isActive }) =>
-                `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
-                  isActive ? 'bg-gray-3000 dark:bg-[#28292E]' : ''
-                }`
-              }
-            >
-              <img
-                src={Info}
-                alt="icon"
-                className="ml-2 w-5 filter dark:invert"
-              />
-              <p className="my-auto pr-1 text-sm">{t('about')}</p>
-            </NavLink>
-            <div className="flex items-center justify-evenly gap-1 px-1">
-              <NavLink
-                target="_blank"
-                to={'https://discord.gg/WHJdfbQDR4'}
-                className={
-                  'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                }
-              >
-                <img
-                  src={Discord}
-                  alt="discord"
-                  className="m-2 w-6 self-center filter dark:invert"
-                />
-              </NavLink>
-              <NavLink
-                target="_blank"
-                to={'https://twitter.com/docsgptai'}
-                className={
-                  'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                }
-              >
-                <img
-                  src={Twitter}
-                  alt="x"
-                  className="m-2 w-5 self-center filter dark:invert"
-                />
-              </NavLink>
-              <NavLink
-                target="_blank"
-                to={'https://github.com/arc53/docsgpt'}
-                className={
-                  'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                }
-              >
-                <img
-                  src={Github}
-                  alt="github"
-                  className="m-2 w-6 self-center filter dark:invert"
-                />
-              </NavLink>
+          <div className="flex flex-col justify-end text-eerie-black dark:text-white">
+            <div className="flex justify-between items-center py-1">
+              <Help />
+
+              <div className="flex items-center gap-1 pr-4">
+                <NavLink
+                  target="_blank"
+                  to={'https://discord.gg/WHJdfbQDR4'}
+                  className={
+                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
+                  }
+                >
+                  <img
+                    src={Discord}
+                    alt="discord"
+                    className="m-2 w-6 self-center filter dark:invert"
+                  />
+                </NavLink>
+                <NavLink
+                  target="_blank"
+                  to={'https://twitter.com/docsgptai'}
+                  className={
+                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
+                  }
+                >
+                  <img
+                    src={Twitter}
+                    alt="x"
+                    className="m-2 w-5 self-center filter dark:invert"
+                  />
+                </NavLink>
+                <NavLink
+                  target="_blank"
+                  to={'https://github.com/arc53/docsgpt'}
+                  className={
+                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
+                  }
+                >
+                  <img
+                    src={Github}
+                    alt="github"
+                    className="m-2 w-6 self-center filter dark:invert"
+                  />
+                </NavLink>
+              </div>
             </div>
           </div>
         </div>
       </div>
       <div className="sticky z-10 h-16 w-full border-b-2 bg-gray-50 dark:border-b-purple-taupe dark:bg-chinese-black md:hidden">
-        <button
-          className="mt-5 ml-6 h-6 w-6 md:hidden"
-          onClick={() => setNavOpen(true)}
-        >
-          <img
-            src={Hamburger}
-            alt="menu toggle"
-            className="w-7 filter dark:invert"
-          />
-        </button>
+        <div className="flex gap-6 items-center h-full ml-6 ">
+          <button
+            className=" h-6 w-6 md:hidden"
+            onClick={() => setNavOpen(true)}
+          >
+            <img
+              src={Hamburger}
+              alt="menu toggle"
+              className="w-7 filter dark:invert"
+            />
+          </button>
+          <div className="text-[#949494] font-medium text-[20px]">DocsGPT</div>
+        </div>
       </div>
       <APIKeyModal
         modalState={apiKeyModalState}

frontend/src/assets/caret-sort.svg

@@ -0,0 +1 @@
+<svg width="18" height="18" viewBox="0 0 15 15" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M4.93179 5.43179C4.75605 5.60753 4.75605 5.89245 4.93179 6.06819C5.10753 6.24392 5.39245 6.24392 5.56819 6.06819L7.49999 4.13638L9.43179 6.06819C9.60753 6.24392 9.89245 6.24392 10.0682 6.06819C10.2439 5.89245 10.2439 5.60753 10.0682 5.43179L7.81819 3.18179C7.73379 3.0974 7.61933 3.04999 7.49999 3.04999C7.38064 3.04999 7.26618 3.0974 7.18179 3.18179L4.93179 5.43179ZM10.0682 9.56819C10.2439 9.39245 10.2439 9.10753 10.0682 8.93179C9.89245 8.75606 9.60753 8.75606 9.43179 8.93179L7.49999 10.8636L5.56819 8.93179C5.39245 8.75606 5.10753 8.75606 4.93179 8.93179C4.75605 9.10753 4.75605 9.39245 4.93179 9.56819L7.18179 11.8182C7.35753 11.9939 7.64245 11.9939 7.81819 11.8182L10.0682 9.56819Z" fill="gray" fill-rule="evenodd" clip-rule="evenodd"></path></svg>

frontend/src/assets/discord.svg

@@ -1 +1,3 @@
-<svg fill="none" width="20" height="20" viewBox="0 0 22 20" xmlns="http://www.w3.org/2000/svg" xml:space="preserve"><path d="M18.942 5.556a16.299 16.299 0 0 0-4.126-1.297c-.178.321-.385.754-.529 1.097a15.175 15.175 0 0 0-4.573 0 11.583 11.583 0 0 0-.535-1.097 16.274 16.274 0 0 0-4.129 1.3c-2.611 3.946-3.319 7.794-2.965 11.587a16.494 16.494 0 0 0 5.061 2.593 12.65 12.65 0 0 0 1.084-1.785 10.689 10.689 0 0 1-1.707-.831c.143-.106.283-.217.418-.331 3.291 1.539 6.866 1.539 10.118 0 .137.114.277.225.418.331-.541.326-1.114.606-1.71.832a12.52 12.52 0 0 0 1.084 1.785 16.46 16.46 0 0 0 5.064-2.595c.415-4.396-.709-8.209-2.973-11.589zM8.678 14.813c-.988 0-1.798-.922-1.798-2.045s.793-2.047 1.798-2.047 1.815.922 1.798 2.047c.001 1.123-.793 2.045-1.798 2.045zm6.644 0c-.988 0-1.798-.922-1.798-2.045s.793-2.047 1.798-2.047 1.815.922 1.798 2.047c0 1.123-.793 2.045-1.798 2.045z" fill="black" fill-opacity="0.54"/></svg>
+<svg width="24" height="25" viewBox="0 0 24 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M18.7359 5.33456C17.4059 4.70233 15.9659 4.24345 14.4659 3.97832C14.4528 3.97789 14.4397 3.98041 14.4276 3.98569C14.4155 3.99098 14.4047 3.9989 14.3959 4.00891C14.2159 4.34542 14.0059 4.78391 13.8659 5.12042C12.2749 4.87568 10.6569 4.87568 9.06594 5.12042C8.92594 4.77371 8.71594 4.34542 8.52594 4.00891C8.51594 3.98852 8.48594 3.97832 8.45594 3.97832C6.95594 4.24345 5.52594 4.70233 4.18594 5.33456C4.17594 5.33456 4.16594 5.34476 4.15594 5.35495C1.43594 9.50526 0.685937 13.5434 1.05594 17.5407C1.05594 17.5611 1.06594 17.5815 1.08594 17.5917C2.88594 18.9378 4.61594 19.7535 6.32594 20.294C6.35594 20.3042 6.38594 20.294 6.39594 20.2736C6.79594 19.7128 7.15594 19.1213 7.46594 18.4993C7.48594 18.4585 7.46594 18.4177 7.42594 18.4075C6.85594 18.1832 6.31594 17.918 5.78594 17.6121C5.74594 17.5917 5.74594 17.5305 5.77594 17.4999C5.88594 17.4184 5.99594 17.3266 6.10594 17.245C6.12594 17.2246 6.15594 17.2246 6.17594 17.2348C9.61594 18.8358 13.3259 18.8358 16.7259 17.2348C16.7459 17.2246 16.7759 17.2246 16.7959 17.245C16.9059 17.3368 17.0159 17.4184 17.1259 17.5101C17.1659 17.5407 17.1659 17.6019 17.1159 17.6223C16.5959 17.9384 16.0459 18.1934 15.4759 18.4177C15.4359 18.4279 15.4259 18.4789 15.4359 18.5095C15.7559 19.1315 16.1159 19.723 16.5059 20.2838C16.5359 20.294 16.5659 20.3042 16.5959 20.294C18.3159 19.7535 20.0459 18.9378 21.8459 17.5917C21.8659 17.5815 21.8759 17.5611 21.8759 17.5407C22.3159 12.9214 21.1459 8.91381 18.7759 5.35495C18.7659 5.34476 18.7559 5.33456 18.7359 5.33456ZM7.98594 15.1036C6.95594 15.1036 6.09594 14.1348 6.09594 12.9417C6.09594 11.7487 6.93594 10.7799 7.98594 10.7799C9.04594 10.7799 9.88594 11.7589 9.87594 12.9417C9.87594 14.1348 9.03594 15.1036 7.98594 15.1036ZM14.9559 15.1036C13.9259 15.1036 13.0659 14.1348 13.0659 12.9417C13.0659 11.7487 13.9059 10.7799 14.9559 10.7799C16.0159 10.7799 16.8559 11.7589 16.8459 12.9417C16.8459 14.1348 16.0159 15.1036 14.9559 15.1036Z" fill="#747474"/>
+</svg>

frontend/src/assets/documentation-dark.svg

@@ -1,3 +1,4 @@
-<svg xmlns="http://www.w3.org/2000/svg" x="0px" y="0px" width="100" height="100" viewBox="0,0,256,256">
-    <g transform="translate(-19.2,-19.2) scale(1.15,1.15)"><g fill="#949494" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M13.172,2h-7.172c-1.1,0 -2,0.9 -2,2v16c0,1.1 0.9,2 2,2h12c1.1,0 2,-0.9 2,-2v-11.172c0,-0.53 -0.211,-1.039 -0.586,-1.414l-4.828,-4.828c-0.375,-0.375 -0.884,-0.586 -1.414,-0.586zM15,18h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM15,14h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM13,9v-5.5l5.5,5.5z"></path></g></g></g>
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16.25 8.64258V16.25C16.25 16.7473 16.0525 17.2242 15.7008 17.5758C15.3492 17.9275 14.8723 18.125 14.375 18.125H5.625C5.12772 18.125 4.65081 17.9275 4.29917 17.5758C3.94754 17.2242 3.75 16.7473 3.75 16.25V3.75C3.75 3.25272 3.94754 2.77581 4.29917 2.42417C4.65081 2.07254 5.12772 1.875 5.625 1.875H9.48242C9.81383 1.87505 10.1316 2.0067 10.366 2.24102L15.884 7.75898C16.1183 7.99335 16.2499 8.31117 16.25 8.64258Z" stroke="#ECECF1" stroke-width="1.11111" stroke-linejoin="round"/>
+<path d="M10 2.1875V6.875C10 7.20652 10.1317 7.52446 10.3661 7.75888C10.6005 7.9933 10.9185 8.125 11.25 8.125H15.9375M6.875 11.25H13.125M6.875 14.375H13.125" stroke="#ECECF1" stroke-width="1.11111" stroke-linecap="round" stroke-linejoin="round"/>
 </svg>

frontend/src/assets/documentation.svg

@@ -1,3 +1,4 @@
-<svg xmlns="http://www.w3.org/2000/svg" x="0px" y="0px" width="100" height="100" viewBox="0,0,256,256">
-    <g transform="translate(-19.2,-19.2) scale(1.15,1.15)"><g fill="black" fill-opacity="0.54" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M13.172,2h-7.172c-1.1,0 -2,0.9 -2,2v16c0,1.1 0.9,2 2,2h12c1.1,0 2,-0.9 2,-2v-11.172c0,-0.53 -0.211,-1.039 -0.586,-1.414l-4.828,-4.828c-0.375,-0.375 -0.884,-0.586 -1.414,-0.586zM15,18h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM15,14h-6c-0.552,0 -1,-0.448 -1,-1v0c0,-0.552 0.448,-1 1,-1h6c0.552,0 1,0.448 1,1v0c0,0.552 -0.448,1 -1,1zM13,9v-5.5l5.5,5.5z"></path></g></g></g>
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16.25 8.64258V16.25C16.25 16.7473 16.0525 17.2242 15.7008 17.5758C15.3492 17.9275 14.8723 18.125 14.375 18.125H5.625C5.12772 18.125 4.65081 17.9275 4.29917 17.5758C3.94754 17.2242 3.75 16.7473 3.75 16.25V3.75C3.75 3.25272 3.94754 2.77581 4.29917 2.42417C4.65081 2.07254 5.12772 1.875 5.625 1.875H9.48242C9.81383 1.87505 10.1316 2.0067 10.366 2.24102L15.884 7.75898C16.1183 7.99335 16.2499 8.31117 16.25 8.64258Z" stroke="#6E6E6E" stroke-width="1.11111" stroke-linejoin="round"/>
+<path d="M10 2.1875V6.875C10 7.20652 10.1317 7.52446 10.3661 7.75888C10.6005 7.9933 10.9185 8.125 11.25 8.125H15.9375M6.875 11.25H13.125M6.875 14.375H13.125" stroke="#6E6E6E" stroke-width="1.11111" stroke-linecap="round" stroke-linejoin="round"/>
 </svg>

frontend/src/assets/envelope-dark.svg

@@ -0,0 +1,3 @@
+<svg width="18" height="18" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M3.04867 15.49H15.1775C16.5339 15.49 17.3172 14.7067 17.3172 13.1548V4.83755C17.3172 3.29309 16.5265 2.50977 14.9515 2.50977H2.82302C1.47463 2.50977 0.683594 3.28569 0.683594 4.83755V13.1545C0.683594 14.7138 1.48138 15.49 3.04867 15.49ZM8.10441 9.3803L2.41609 3.76816C2.58163 3.70034 2.77738 3.66273 3.01106 3.66273H14.9971C15.2305 3.66273 15.434 3.70034 15.6072 3.78327L9.92692 9.38062C9.60291 9.7043 9.31652 9.84766 9.01534 9.84766C8.71384 9.84766 8.42841 9.70398 8.10441 9.3803ZM1.83559 13.1548V4.76234L6.16717 9.01162L1.84331 13.2824C1.83592 13.2448 1.83559 13.1998 1.83559 13.1548ZM16.1646 4.84494V13.2599L11.8629 9.0113L16.1649 4.78516L16.1646 4.84494ZM3.01106 14.3377C2.79249 14.3377 2.61184 14.3075 2.4537 14.2397L6.95852 9.78723L7.44838 10.2694C7.97552 10.7891 8.48017 11.0077 9.01534 11.0077C9.54249 11.0077 10.0548 10.7891 10.5823 10.2694L11.0718 9.78723L15.5696 14.2319C15.4111 14.3075 15.2154 14.3374 14.9968 14.3374L3.01106 14.3377Z" fill="#ECECF1"/>
+</svg>

frontend/src/assets/envelope.svg

@@ -0,0 +1,3 @@
+<svg width="18" height="18" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M3.04867 15.49H15.1775C16.5339 15.49 17.3172 14.7067 17.3172 13.1548V4.83755C17.3172 3.29309 16.5265 2.50977 14.9515 2.50977H2.82302C1.47463 2.50977 0.683594 3.28569 0.683594 4.83755V13.1545C0.683594 14.7138 1.48138 15.49 3.04867 15.49ZM8.10441 9.3803L2.41609 3.76816C2.58163 3.70034 2.77738 3.66273 3.01106 3.66273H14.9971C15.2305 3.66273 15.434 3.70034 15.6072 3.78327L9.92692 9.38062C9.60291 9.7043 9.31652 9.84766 9.01534 9.84766C8.71384 9.84766 8.42841 9.70398 8.10441 9.3803ZM1.83559 13.1548V4.76234L6.16717 9.01162L1.84331 13.2824C1.83592 13.2448 1.83559 13.1998 1.83559 13.1548ZM16.1646 4.84494V13.2599L11.8629 9.0113L16.1649 4.78516L16.1646 4.84494ZM3.01106 14.3377C2.79249 14.3377 2.61184 14.3075 2.4537 14.2397L6.95852 9.78723L7.44838 10.2694C7.97552 10.7891 8.48017 11.0077 9.01534 11.0077C9.54249 11.0077 10.0548 10.7891 10.5823 10.2694L11.0718 9.78723L15.5696 14.2319C15.4111 14.3075 15.2154 14.3374 14.9968 14.3374L3.01106 14.3377Z" fill="#6F6F6F"/>
+</svg>

frontend/src/assets/openNewChat.svg

@@ -0,0 +1,4 @@
+<svg width="21" height="20" viewBox="0 0 21 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="1.66663" y="1" width="18" height="18" rx="9" stroke="#747474" stroke-width="2"/>
+<path d="M9.89252 14.1268C9.89252 14.3321 9.97405 14.5289 10.1192 14.674C10.2643 14.8191 10.4611 14.9006 10.6663 14.9006C10.8716 14.9006 11.0684 14.8191 11.2135 14.674C11.3586 14.5289 11.4401 14.3321 11.4401 14.1268L11.4401 10.7737L14.7933 10.7737C14.9985 10.7737 15.1953 10.6922 15.3404 10.547C15.4856 10.4019 15.5671 10.2051 15.5671 9.99989C15.5671 9.79466 15.4856 9.59784 15.3404 9.45272C15.1953 9.30761 14.9985 9.22608 14.7933 9.22608L11.4401 9.22608L11.4401 5.87293C11.4401 5.6677 11.3586 5.47088 11.2135 5.32576C11.0684 5.18065 10.8716 5.09912 10.6663 5.09912C10.4611 5.09912 10.2643 5.18065 10.1192 5.32576C9.97405 5.47088 9.89252 5.6677 9.89252 5.87293L9.89252 9.22608L6.53937 9.22608C6.33414 9.22608 6.13732 9.30761 5.99221 9.45272C5.84709 9.59784 5.76556 9.79466 5.76556 9.99989C5.76556 10.2051 5.84709 10.4019 5.99221 10.547C6.13732 10.6922 6.33414 10.7737 6.53937 10.7737L9.89252 10.7737L9.89252 14.1268Z" fill="#747474"/>
+</svg>

frontend/src/components/Help.tsx

@@ -0,0 +1,80 @@
+import { useState, useRef, useEffect } from 'react';
+import Info from '../assets/info.svg';
+import PageIcon from '../assets/documentation.svg';
+import EmailIcon from '../assets/envelope.svg';
+import { useTranslation } from 'react-i18next';
+const Help = () => {
+  const [isOpen, setIsOpen] = useState(false);
+  const dropdownRef = useRef<HTMLDivElement | null>(null);
+  const buttonRef = useRef<HTMLButtonElement | null>(null);
+  const { t } = useTranslation();
+
+  const toggleDropdown = () => {
+    setIsOpen((prev) => !prev);
+  };
+
+  const handleClickOutside = (event: MouseEvent) => {
+    if (
+      dropdownRef.current &&
+      !dropdownRef.current.contains(event.target as Node) &&
+      buttonRef.current &&
+      !buttonRef.current.contains(event.target as Node)
+    ) {
+      setIsOpen(false);
+    }
+  };
+
+  useEffect(() => {
+    document.addEventListener('mousedown', handleClickOutside);
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+    };
+  }, []);
+
+  return (
+    <div className="relative inline-block text-sm" ref={dropdownRef}>
+      <button
+        ref={buttonRef}
+        onClick={toggleDropdown}
+        className="my-auto mx-4 w-full flex items-center h-9 gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E]"
+      >
+        <img src={Info} alt="info" className="ml-2 w-5 filter dark:invert" />
+        {t('help')}
+      </button>
+      {isOpen && (
+        <div
+          className={`absolute translate-x-4 -translate-y-28 z-10 w-48 shadow-lg bg-white dark:bg-[#444654] rounded-xl`}
+        >
+          <a
+            href="https://docs.docsgpt.cloud/"
+            target="_blank"
+            rel="noopener noreferrer"
+            className="flex items-start gap-4 px-4 py-2 text-black dark:text-white hover:bg-bright-gray dark:hover:bg-[#545561] rounded-t-xl"
+          >
+            <img
+              src={PageIcon}
+              alt="Documentation"
+              className="filter dark:invert"
+              width={20}
+            />
+            {t('documentation')}
+          </a>
+          <a
+            href="mailto:support@docsgpt.cloud"
+            className="flex items-start gap-4 px-4 py-2 text-black dark:text-white hover:bg-bright-gray dark:hover:bg-[#545561] rounded-b-xl"
+          >
+            <img
+              src={EmailIcon}
+              alt="Email Us"
+              className="filter dark:invert p-0.5"
+              width={20}
+            />
+            {t('emailUs')}
+          </a>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default Help;

frontend/src/components/ShareButton.tsx

@@ -0,0 +1,36 @@
+import { useState } from 'react';
+import ShareIcon from '../assets/share.svg';
+import { ShareConversationModal } from '../modals/ShareConversationModal';
+
+type ShareButtonProps = {
+  conversationId: string;
+};
+
+export default function ShareButton({ conversationId }: ShareButtonProps) {
+  const [isShareModalOpen, setShareModalState] = useState<boolean>(false);
+  return (
+    <>
+      <button
+        title="Share"
+        onClick={() => {
+          setShareModalState(true);
+        }}
+        className="absolute top-4 right-20 z-20 rounded-full hover:bg-bright-gray dark:hover:bg-[#28292E]"
+      >
+        <img
+          className="m-2 h-5 w-5 filter dark:invert"
+          alt="share"
+          src={ShareIcon}
+        />
+      </button>
+      {isShareModalOpen && (
+        <ShareConversationModal
+          close={() => {
+            setShareModalState(false);
+          }}
+          conversationId={conversationId}
+        />
+      )}
+    </>
+  );
+}

frontend/src/components/SkeletonLoader.tsx

@@ -0,0 +1,138 @@
+import React, { useState, useEffect } from 'react';
+
+interface SkeletonLoaderProps {
+  count?: number;
+  component?: 'default' | 'analysis' | 'chatbot' | 'logs';
+}
+
+const SkeletonLoader: React.FC<SkeletonLoaderProps> = ({
+  count = 1,
+  component = 'default',
+}) => {
+  const [skeletonCount, setSkeletonCount] = useState(count);
+
+  useEffect(() => {
+    const handleResize = () => {
+      const windowWidth = window.innerWidth;
+
+      if (windowWidth > 1024) {
+        setSkeletonCount(1);
+      } else if (windowWidth > 768) {
+        setSkeletonCount(count);
+      } else {
+        setSkeletonCount(Math.min(count, 2));
+      }
+    };
+
+    handleResize();
+    window.addEventListener('resize', handleResize);
+
+    return () => {
+      window.removeEventListener('resize', handleResize);
+    };
+  }, [count]);
+
+  return (
+    <div className="flex flex-col space-y-4">
+      {component === 'default' ? (
+        [...Array(skeletonCount)].map((_, idx) => (
+          <div
+            key={idx}
+            className={`p-6 ${skeletonCount === 1 ? 'w-full' : 'w-60'} dark:bg-raisin-black rounded-3xl animate-pulse`}
+          >
+            <div className="space-y-4">
+              <div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-3/4"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-5/6"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-1/2"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-3/4"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-full"></div>
+              </div>
+              <div className="border-t border-gray-600 my-4"></div>
+              <div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-2/3"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-1/4"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-full"></div>
+              </div>
+              <div className="border-t border-gray-600 my-4"></div>
+              <div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-5/6"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-1/3"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-2/3"></div>
+                <div className="h-4 bg-gray-600 rounded mb-2 w-full"></div>
+              </div>
+              <div className="border-t border-gray-600 my-4"></div>
+              <div className="h-4 bg-gray-600 rounded w-3/4 mb-2"></div>
+              <div className="h-4 bg-gray-600 rounded w-5/6 mb-2"></div>
+            </div>
+          </div>
+        ))
+      ) : component === 'analysis' ? (
+        [...Array(skeletonCount)].map((_, idx) => (
+          <div
+            key={idx}
+            className="p-6 w-full dark:bg-raisin-black rounded-3xl animate-pulse"
+          >
+            <div className="space-y-6">
+              <div className="space-y-2">
+                <div className="h-4 bg-gray-600 rounded w-1/3 mb-4"></div>
+                <div className="grid grid-cols-6 gap-2 items-end">
+                  <div className="h-32 bg-gray-600 rounded"></div>
+                  <div className="h-24 bg-gray-600 rounded"></div>
+                  <div className="h-40 bg-gray-600 rounded"></div>
+                  <div className="h-28 bg-gray-600 rounded"></div>
+                  <div className="h-36 bg-gray-600 rounded"></div>
+                  <div className="h-20 bg-gray-600 rounded"></div>
+                </div>
+              </div>
+              <div className="space-y-2">
+                <div className="h-4 bg-gray-600 rounded w-1/4 mb-4"></div>
+                <div className="h-32 bg-gray-600 rounded"></div>
+              </div>
+              <div className="grid grid-cols-2 gap-4">
+                <div className="h-4 bg-gray-600 rounded w-full"></div>
+                <div className="h-4 bg-gray-600 rounded w-full"></div>
+              </div>
+            </div>
+          </div>
+        ))
+      ) : component === 'chatbot' ? (
+        <div className="space-y-2 p-6 w-full dark:bg-raisin-black rounded-3xl animate-pulse">
+          <div className="grid grid-cols-4 gap-2 p-2">
+            <div className="h-4 bg-gray-600 rounded w-full"></div>
+            <div className="h-4 bg-gray-600 rounded w-full"></div>
+            <div className="h-4 bg-gray-600 rounded w-full"></div>
+            <div className="h-4 bg-gray-600 rounded w-full"></div>
+          </div>
+          <div className="border-t border-gray-600 my-2"></div>
+
+          {[...Array(skeletonCount * 6)].map((_, idx) => (
+            <div key={idx} className="grid grid-cols-4 gap-2 p-2 space-x-2">
+              <div className="h-4 bg-gray-500 rounded w-full"></div>
+              <div className="h-4 bg-gray-500 rounded w-full"></div>
+              <div className="h-4 bg-gray-500 rounded w-full"></div>
+              <div className="h-4 bg-gray-500 rounded w-full"></div>
+            </div>
+          ))}
+        </div>
+      ) : (
+        [...Array(skeletonCount)].map((_, idx) => (
+          <div
+            key={idx}
+            className="p-6 w-full dark:bg-raisin-black rounded-3xl animate-pulse"
+          >
+            <div className="space-y-4">
+              <div className="h-4 bg-gray-600 rounded w-1/2"></div>
+              <div className="h-4 bg-gray-600 rounded w-5/6"></div>
+              <div className="h-4 bg-gray-600 rounded w-3/4"></div>
+              <div className="h-4 bg-gray-600 rounded w-2/3"></div>
+              <div className="h-4 bg-gray-600 rounded w-1/4"></div>
+            </div>
+          </div>
+        ))
+      )}
+    </div>
+  );
+};
+
+export default SkeletonLoader;

frontend/src/components/SourceDropdown.tsx

@@ -121,9 +121,12 @@ function SourceDropdown({
             className="flex cursor-pointer items-center justify-between hover:bg-gray-100 dark:text-bright-gray dark:hover:bg-purple-taupe"
             onClick={handleEmptyDocumentSelect}
           >
-            <span className="ml-4 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3" onClick = {() => {
-              handlePostDocumentSelect(null);
-            }}>
+            <span
+              className="ml-4 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3"
+              onClick={() => {
+                handlePostDocumentSelect(null);
+              }}
+            >
               {t('none')}
             </span>
           </div>

frontend/src/conversation/Conversation.tsx

@@ -1,22 +1,24 @@
 import { Fragment, useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
-
+import newChatIcon from '../assets/openNewChat.svg';
 import ArrowDown from '../assets/arrow-down.svg';
 import Send from '../assets/send.svg';
 import SendDark from '../assets/send_dark.svg';
-import ShareIcon from '../assets/share.svg';
 import SpinnerDark from '../assets/spinner-dark.svg';
 import Spinner from '../assets/spinner.svg';
 import RetryIcon from '../components/RetryIcon';
+import { useNavigate } from 'react-router-dom';
 import Hero from '../Hero';
 import { useDarkTheme, useMediaQuery } from '../hooks';
 import { ShareConversationModal } from '../modals/ShareConversationModal';
+import { setConversation, updateConversationId } from './conversationSlice';
 import { selectConversationId } from '../preferences/preferenceSlice';
 import { AppDispatch } from '../store';
 import ConversationBubble from './ConversationBubble';
 import { handleSendFeedback } from './conversationHandlers';
 import { FEEDBACK, Query } from './conversationModels';
+import ShareIcon from '../assets/share.svg';
 import {
   addQuery,
   fetchAnswer,
@@ -27,6 +29,7 @@ import {
 
 export default function Conversation() {
   const queries = useSelector(selectQueries);
+  const navigate = useNavigate();
   const status = useSelector(selectStatus);
   const conversationId = useSelector(selectConversationId);
   const dispatch = useDispatch<AppDispatch>();
@@ -46,6 +49,9 @@ export default function Conversation() {
   };
   useEffect(() => {
     !eventInterrupt && scrollIntoView();
+    if (queries.length == 0) {
+      resetConversation();
+    }
   }, [queries.length, queries[queries.length - 1]]);
 
   useEffect(() => {
@@ -120,6 +126,17 @@ export default function Conversation() {
       handleInput();
     }
   };
+  const resetConversation = () => {
+    dispatch(setConversation([]));
+    dispatch(
+      updateConversationId({
+        query: { conversationId: null },
+      }),
+    );
+  };
+  const newChat = () => {
+    if (queries && queries.length > 0) resetConversation();
+  };
 
   const prepResponseView = (query: Query, index: number) => {
     let responseView;
@@ -197,23 +214,41 @@ export default function Conversation() {
     };
   }, []);
   return (
-    <div className="flex flex-col gap-1 h-full justify-end">
-      {conversationId && (
-        <>
+    <div className="flex flex-col gap-1 h-full justify-end ">
+      {conversationId && queries.length > 0 && (
+        <div className="absolute top-4 right-20 z-20 ">
           {' '}
-          <button
-            title="Share"
-            onClick={() => {
-              setShareModalState(true);
-            }}
-            className="absolute top-4 right-20 z-20 rounded-full hover:bg-bright-gray dark:hover:bg-[#28292E]"
-          >
-            <img
-              className="m-2 h-5 w-5 filter dark:invert"
-              alt="share"
-              src={ShareIcon}
-            />
-          </button>
+          <div className="flex items-center gap-4 ">
+            {isMobile && queries.length > 0 && (
+              <button
+                title="Open New Chat"
+                onClick={() => {
+                  newChat();
+                }}
+                className="hover:bg-bright-gray dark:hover:bg-[#28292E]"
+              >
+                <img
+                  className=" h-5 w-5 filter dark:invert "
+                  alt="NewChat"
+                  src={newChatIcon}
+                />
+              </button>
+            )}
+
+            <button
+              title="Share"
+              onClick={() => {
+                setShareModalState(true);
+              }}
+              className=" hover:bg-bright-gray dark:hover:bg-[#28292E]"
+            >
+              <img
+                className=" h-5 w-5 filter dark:invert"
+                alt="share"
+                src={ShareIcon}
+              />
+            </button>
+          </div>
           {isShareModalOpen && (
             <ShareConversationModal
               close={() => {
@@ -222,7 +257,7 @@ export default function Conversation() {
               conversationId={conversationId}
             />
           )}
-        </>
+        </div>
       )}
       <div
         ref={conversationRef}

frontend/src/conversation/ConversationBubble.tsx

@@ -354,7 +354,7 @@ const ConversationBubble = forwardRef<
               >
                 <div>
                   <div
-                    className={`flex items-center justify-center rounded-full p-2 dark:bg-transparent ${
+                    className={`flex items-center justify-center rounded-full p-2 ${
                       isLikeHovered
                         ? 'bg-[#EEEEEE] dark:bg-purple-taupe'
                         : 'bg-[#ffffff] dark:bg-transparent'

frontend/src/conversation/ConversationTile.tsx

@@ -1,4 +1,10 @@
-import { SyntheticEvent, useEffect, useRef, useState } from 'react';
+import {
+  SyntheticEvent,
+  useEffect,
+  useRef,
+  useState,
+  useCallback,
+} from 'react';
 import { useSelector } from 'react-redux';
 import Edit from '../assets/edit.svg';
 import Exit from '../assets/exit.svg';
@@ -75,6 +81,36 @@ export default function ConversationTile({
       document.removeEventListener('mousedown', handleClickOutside);
     };
   }, []);
+
+  const preventScroll = useCallback((event: WheelEvent | TouchEvent) => {
+    event.preventDefault();
+  }, []);
+
+  useEffect(() => {
+    const conversationsMainDiv = document.getElementById(
+      'conversationsMainDiv',
+    );
+
+    if (conversationsMainDiv) {
+      if (isOpen) {
+        conversationsMainDiv.addEventListener('wheel', preventScroll, {
+          passive: false,
+        });
+        conversationsMainDiv.addEventListener('touchmove', preventScroll, {
+          passive: false,
+        });
+      } else {
+        conversationsMainDiv.removeEventListener('wheel', preventScroll);
+        conversationsMainDiv.removeEventListener('touchmove', preventScroll);
+      }
+
+      return () => {
+        conversationsMainDiv.removeEventListener('wheel', preventScroll);
+        conversationsMainDiv.removeEventListener('touchmove', preventScroll);
+      };
+    }
+  }, [isOpen]);
+
   function onClear() {
     setConversationsName(conversation.name);
     setIsEdit(false);
@@ -147,7 +183,7 @@ export default function ConversationTile({
               <button
                 onClick={(event: SyntheticEvent) => {
                   event.stopPropagation();
-                  setOpen(true);
+                  setOpen(!isOpen);
                 }}
                 className="mr-2 flex w-4 justify-center"
               >

frontend/src/conversation/conversationHandlers.ts

@@ -38,9 +38,11 @@ export function handleFetchAnswer(
     prompt_id: promptId,
     chunks: chunks,
     token_limit: token_limit,
+    isNoneDoc: selectedDocs === null,
   };
-  if (selectedDocs && 'id' in selectedDocs)
+  if (selectedDocs && 'id' in selectedDocs) {
     payload.active_docs = selectedDocs.id as string;
+  }
   payload.retriever = selectedDocs?.retriever as string;
   return conversationService
     .answer(payload, signal)
@@ -84,26 +86,16 @@ export function handleFetchAnswerSteaming(
     prompt_id: promptId,
     chunks: chunks,
     token_limit: token_limit,
+    isNoneDoc: selectedDocs === null,
   };
-  if (selectedDocs && 'id' in selectedDocs)
+  if (selectedDocs && 'id' in selectedDocs) {
     payload.active_docs = selectedDocs.id as string;
+  }
   payload.retriever = selectedDocs?.retriever as string;
 
   return new Promise<Answer>((resolve, reject) => {
     conversationService
-      .answerStream(
-        {
-          question: question,
-          active_docs: selectedDocs?.id as string,
-          history: JSON.stringify(history),
-          conversation_id: conversationId,
-          prompt_id: promptId,
-          chunks: chunks,
-          token_limit: token_limit,
-          isNoneDoc: selectedDocs === null,
-        },
-        signal,
-      )
+      .answerStream(payload, signal)
       .then((response) => {
         if (!response.body) throw Error('No response body');
 
@@ -169,20 +161,13 @@ export function handleSearch(
     conversation_id: conversation_id,
     chunks: chunks,
     token_limit: token_limit,
+    isNoneDoc: selectedDocs === null,
   };
   if (selectedDocs && 'id' in selectedDocs)
     payload.active_docs = selectedDocs.id as string;
   payload.retriever = selectedDocs?.retriever as string;
   return conversationService
-    .search({
-      question: question,
-      active_docs: selectedDocs?.id as string,
-      conversation_id,
-      history,
-      chunks: chunks,
-      token_limit: token_limit,
-      isNoneDoc: selectedDocs === null,
-    })
+    .search(payload)
     .then((response) => response.json())
     .then((data) => {
       return data;

frontend/src/conversation/conversationModels.ts

@@ -40,4 +40,5 @@ export interface RetrievalPayload {
   prompt_id?: string | null;
   chunks: string;
   token_limit: number;
+  isNoneDoc: boolean;
 }

frontend/src/index.css

@@ -53,19 +53,19 @@ body.dark {
   }
 
   .table-default th {
-    @apply border-r border-silver dark:border-silver/40 p-4 w-[244px];
+    @apply p-4 w-[244px] font-normal text-gray-400; /* Remove border-r */
   }
 
   .table-default th:last-child {
-    @apply w-[auto] border-r-0;
+    @apply w-[auto];
   }
 
   .table-default td {
-    @apply border-r border-t border-silver dark:border-silver/40 px-4 py-2;
+    @apply border-t border-silver dark:border-silver/40 px-4 py-2; /* Remove border-r */
   }
 
   .table-default td:last-child {
-    @apply border-r-0;
+    @apply border-r-0; /* Ensure no right border on the last column */
   }
 }
 

frontend/src/locale/en.json

@@ -10,6 +10,9 @@
   "sourceDocs": "Source",
   "none": "None",
   "cancel": "Cancel",
+  "help": "Help",
+  "emailUs": "Email us",
+  "documentation": "documentation",
   "demo": [
     {
       "header": "Learn about DocsGPT",
@@ -83,7 +86,7 @@
       "start": "Start Chatting",
       "name": "Name",
       "choose": "Choose Files",
-      "info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .zip limited to 25mb",
+      "info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .zip limited to 25mb",
       "uploadedFiles": "Uploaded Files",
       "cancel": "Cancel",
       "train": "Train",

frontend/src/locale/es.json

@@ -10,6 +10,9 @@
   "sourceDocs": "Fuente",
   "none": "Nada",
   "cancel": "Cancelar",
+  "help": "Asistencia",
+  "emailUs": "Envíanos un correo",
+  "documentation": "documentación",
   "demo": [
     {
       "header": "Aprende sobre DocsGPT",
@@ -83,7 +86,7 @@
       "start": "Empezar a chatear",
       "name": "Nombre",
       "choose": "Seleccionar Archivos",
-      "info": "Por favor, suba archivos .pdf, .txt, .rst, .docx, .md, .zip limitados a 25 MB",
+      "info": "Por favor, suba archivos .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .zip limitados a 25 MB",
       "uploadedFiles": "Archivos Subidos",
       "cancel": "Cancelar",
       "train": "Entrenar",

frontend/src/locale/i18n.ts

@@ -25,7 +25,7 @@ i18n
       zh: {
         translation: zh,
       },
-      "zh-TW": {
+      'zh-TW': {
         translation: zhTW,
       },
     },

frontend/src/locale/jp.json

@@ -10,6 +10,9 @@
   "sourceDocs": "ソース",
   "none": "なし",
   "cancel": "キャンセル",
+  "help": "ヘルプ",
+  "emailUs": "メールを送る",
+  "documentation": "ドキュメント",
   "demo": [
     {
       "header": "DocsGPTについて学ぶ",
@@ -83,7 +86,7 @@
       "start": "チャットを開始する",
       "name": "名前",
       "choose": "ファイルを選択",
-      "info": ".pdf, .txt, .rst, .docx, .md, .zipファイルを25MBまでアップロードしてください",
+      "info": ".pdf, .txt, .rst, .docx, .md, .json, .zipファイルを25MBまでアップロードしてください",
       "uploadedFiles": "アップロードされたファイル",
       "cancel": "キャンセル",
       "train": "トレーニング",

frontend/src/locale/zh-TW.json

@@ -10,6 +10,9 @@
   "sourceDocs": "原始文件",
   "none": "無",
   "cancel": "取消",
+  "help": "聯繫支援",
+  "emailUs": "寄送電子郵件給我們",
+  "documentation": "文件",
   "demo": [
     {
       "header": "了解 DocsGPT",
@@ -77,7 +80,7 @@
       "remote": "遠端",
       "name": "名稱",
       "choose": "選擇檔案",
-      "info": "請上傳 .pdf, .txt, .rst, .docx, .md, .zip 檔案，大小限制為 25MB",
+      "info": "請上傳 .pdf, .txt, .rst, .docx, .md, .json, .zip 檔案，大小限制為 25MB",
       "uploadedFiles": "已上傳的檔案",
       "cancel": "取消",
       "train": "訓練",

frontend/src/locale/zh.json

@@ -10,6 +10,9 @@
   "sourceDocs": "源",
   "none": "无",
   "cancel": "取消",
+  "help": "联系支持",
+  "emailUs": "给我们发邮件",
+  "documentation": "文档",
   "demo": [
     {
       "header": "了解 DocsGPT",
@@ -83,7 +86,7 @@
       "start": "开始聊天",
       "name": "名称",
       "choose": "选择文件",
-      "info": "请上传 .pdf, .txt, .rst, .docx, .md, .zip 文件，限 25MB",
+      "info": "请上传 .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .zip 文件，限 25MB",
       "uploadedFiles": "已上传文件",
       "cancel": "取消",
       "train": "训练",

frontend/src/settings/APIKeys.tsx

@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { useState } from 'react';
 import { useTranslation } from 'react-i18next';
 
 import userService from '../api/services/userService';
@@ -6,6 +6,7 @@ import Trash from '../assets/trash.svg';
 import CreateAPIKeyModal from '../modals/CreateAPIKeyModal';
 import SaveAPIKeyModal from '../modals/SaveAPIKeyModal';
 import { APIKeyData } from './types';
+import SkeletonLoader from '../components/SkeletonLoader';
 
 export default function APIKeys() {
   const { t } = useTranslation();
@@ -13,8 +14,10 @@ export default function APIKeys() {
   const [isSaveKeyModalOpen, setSaveKeyModal] = React.useState(false);
   const [newKey, setNewKey] = React.useState('');
   const [apiKeys, setApiKeys] = React.useState<APIKeyData[]>([]);
+  const [loading, setLoading] = useState(true);
 
   const handleFetchKeys = async () => {
+    setLoading(true);
     try {
       const response = await userService.getAPIKeys();
       if (!response.ok) {
@@ -24,6 +27,8 @@ export default function APIKeys() {
       setApiKeys(apiKeys);
     } catch (error) {
       console.log(error);
+    } finally {
+      setLoading(false);
     }
   };
 
@@ -37,7 +42,7 @@ export default function APIKeys() {
         return response.json();
       })
       .then((data) => {
-        data.status === 'ok' &&
+        data.success === true &&
           setApiKeys((previous) => previous.filter((elem) => elem.id !== id));
       })
       .catch((error) => {
@@ -75,6 +80,7 @@ export default function APIKeys() {
   React.useEffect(() => {
     handleFetchKeys();
   }, []);
+
   return (
     <div className="mt-8">
       <div className="flex flex-col max-w-[876px]">
@@ -100,41 +106,45 @@ export default function APIKeys() {
         )}
         <div className="mt-[27px] w-full">
           <div className="w-full overflow-x-auto">
-            <table className="table-default">
-              <thead>
-                <tr>
-                  <th>{t('settings.apiKeys.name')}</th>
-                  <th>{t('settings.apiKeys.sourceDoc')}</th>
-                  <th>{t('settings.apiKeys.key')}</th>
-                  <th></th>
-                </tr>
-              </thead>
-              <tbody>
-                {!apiKeys?.length && (
+            {loading ? (
+              <SkeletonLoader count={1} component={'chatbot'} />
+            ) : (
+              <table className="table-default">
+                <thead>
                   <tr>
-                    <td colSpan={4} className="!p-4">
-                      {t('settings.apiKeys.noData')}
-                    </td>
+                    <th>{t('settings.apiKeys.name')}</th>
+                    <th>{t('settings.apiKeys.sourceDoc')}</th>
+                    <th>{t('settings.apiKeys.key')}</th>
+                    <th></th>
                   </tr>
-                )}
-                {apiKeys?.map((element, index) => (
-                  <tr key={index}>
-                    <td>{element.name}</td>
-                    <td>{element.source}</td>
-                    <td>{element.key}</td>
-                    <td>
-                      <img
-                        src={Trash}
-                        alt="Delete"
-                        className="h-4 w-4 cursor-pointer hover:opacity-50"
-                        id={`img-${index}`}
-                        onClick={() => handleDeleteKey(element.id)}
-                      />
-                    </td>
-                  </tr>
-                ))}
-              </tbody>
-            </table>
+                </thead>
+                <tbody>
+                  {!apiKeys?.length && (
+                    <tr>
+                      <td colSpan={4} className="!p-4">
+                        {t('settings.apiKeys.noData')}
+                      </td>
+                    </tr>
+                  )}
+                  {apiKeys?.map((element, index) => (
+                    <tr key={index}>
+                      <td>{element.name}</td>
+                      <td>{element.source}</td>
+                      <td>{element.key}</td>
+                      <td>
+                        <img
+                          src={Trash}
+                          alt="Delete"
+                          className="h-4 w-4 cursor-pointer hover:opacity-50"
+                          id={`img-${index}`}
+                          onClick={() => handleDeleteKey(element.id)}
+                        />
+                      </td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            )}
           </div>
         </div>
       </div>

frontend/src/settings/Analytics.tsx

@@ -1,3 +1,4 @@
+import React, { useState, useEffect } from 'react';
 import {
   BarElement,
   CategoryScale,
@@ -7,7 +8,6 @@ import {
   Title,
   Tooltip,
 } from 'chart.js';
-import React from 'react';
 import { Bar } from 'react-chartjs-2';
 
 import userService from '../api/services/userService';
@@ -17,6 +17,8 @@ import { formatDate } from '../utils/dateTimeUtils';
 import { APIKeyData } from './types';
 
 import type { ChartData } from 'chart.js';
+import SkeletonLoader from '../components/SkeletonLoader';
+
 ChartJS.register(
   CategoryScale,
   LinearScale,
@@ -35,37 +37,37 @@ const filterOptions = [
 ];
 
 export default function Analytics() {
-  const [messagesData, setMessagesData] = React.useState<Record<
+  const [messagesData, setMessagesData] = useState<Record<
     string,
     number
   > | null>(null);
-  const [tokenUsageData, setTokenUsageData] = React.useState<Record<
+  const [tokenUsageData, setTokenUsageData] = useState<Record<
     string,
     number
   > | null>(null);
-  const [feedbackData, setFeedbackData] = React.useState<Record<
+  const [feedbackData, setFeedbackData] = useState<Record<
     string,
-    {
-      positive: number;
-      negative: number;
-    }
+    { positive: number; negative: number }
   > | null>(null);
-  const [chatbots, setChatbots] = React.useState<APIKeyData[]>([]);
-  const [selectedChatbot, setSelectedChatbot] =
-    React.useState<APIKeyData | null>();
-  const [messagesFilter, setMessagesFilter] = React.useState<{
+  const [chatbots, setChatbots] = useState<APIKeyData[]>([]);
+  const [selectedChatbot, setSelectedChatbot] = useState<APIKeyData | null>();
+  const [messagesFilter, setMessagesFilter] = useState<{
     label: string;
     value: string;
   }>({ label: '30 Days', value: 'last_30_days' });
-  const [tokenUsageFilter, setTokenUsageFilter] = React.useState<{
+  const [tokenUsageFilter, setTokenUsageFilter] = useState<{
     label: string;
     value: string;
   }>({ label: '30 Days', value: 'last_30_days' });
-  const [feedbackFilter, setFeedbackFilter] = React.useState<{
+  const [feedbackFilter, setFeedbackFilter] = useState<{
     label: string;
     value: string;
   }>({ label: '30 Days', value: 'last_30_days' });
 
+  const [loadingMessages, setLoadingMessages] = useState(true);
+  const [loadingTokens, setLoadingTokens] = useState(true);
+  const [loadingFeedback, setLoadingFeedback] = useState(true);
+
   const fetchChatbots = async () => {
     try {
       const response = await userService.getAPIKeys();
@@ -80,6 +82,7 @@ export default function Analytics() {
   };
 
   const fetchMessagesData = async (chatbot_id?: string, filter?: string) => {
+    setLoadingMessages(true);
     try {
       const response = await userService.getMessageAnalytics({
         api_key_id: chatbot_id,
@@ -92,10 +95,13 @@ export default function Analytics() {
       setMessagesData(data.messages);
     } catch (error) {
       console.error(error);
+    } finally {
+      setLoadingMessages(false);
     }
   };
 
   const fetchTokenData = async (chatbot_id?: string, filter?: string) => {
+    setLoadingTokens(true);
     try {
       const response = await userService.getTokenAnalytics({
         api_key_id: chatbot_id,
@@ -108,10 +114,13 @@ export default function Analytics() {
       setTokenUsageData(data.token_usage);
     } catch (error) {
       console.error(error);
+    } finally {
+      setLoadingTokens(false);
     }
   };
 
   const fetchFeedbackData = async (chatbot_id?: string, filter?: string) => {
+    setLoadingFeedback(true);
     try {
       const response = await userService.getFeedbackAnalytics({
         api_key_id: chatbot_id,
@@ -124,30 +133,33 @@ export default function Analytics() {
       setFeedbackData(data.feedback);
     } catch (error) {
       console.error(error);
+    } finally {
+      setLoadingFeedback(false);
     }
   };
 
-  React.useEffect(() => {
+  useEffect(() => {
     fetchChatbots();
   }, []);
 
-  React.useEffect(() => {
+  useEffect(() => {
     const id = selectedChatbot?.id;
     const filter = messagesFilter;
     fetchMessagesData(id, filter?.value);
   }, [selectedChatbot, messagesFilter]);
 
-  React.useEffect(() => {
+  useEffect(() => {
     const id = selectedChatbot?.id;
     const filter = tokenUsageFilter;
     fetchTokenData(id, filter?.value);
   }, [selectedChatbot, tokenUsageFilter]);
 
-  React.useEffect(() => {
+  useEffect(() => {
     const id = selectedChatbot?.id;
     const filter = feedbackFilter;
     fetchFeedbackData(id, filter?.value);
   }, [selectedChatbot, feedbackFilter]);
+
   return (
     <div className="mt-12">
       <div className="flex flex-col items-start">
@@ -181,8 +193,10 @@ export default function Analytics() {
             border="border"
           />
         </div>
+
+        {/* Messages Analytics */}
         <div className="mt-8 w-full flex flex-col [@media(min-width:1080px)]:flex-row gap-3">
-          <div className="h-[345px] [@media(min-width:1080px)]:w-1/2 w-full px-6 py-5 border rounded-2xl border-silver dark:border-silver/40">
+          <div className="h-[345px] [@media(min-width:1080px)]:w-1/2 w-full px-6 py-5 border rounded-2xl border-silver dark:border-silver/40 overflow-hidden">
             <div className="flex flex-row items-center justify-start gap-3">
               <p className="font-bold text-jet dark:text-bright-gray">
                 Messages
@@ -208,26 +222,32 @@ export default function Analytics() {
                 id="legend-container-1"
                 className="flex flex-row items-center justify-end"
               ></div>
-              <AnalyticsChart
-                data={{
-                  labels: Object.keys(messagesData || {}).map((item) =>
-                    formatDate(item),
-                  ),
-                  datasets: [
-                    {
-                      label: 'Messages',
-                      data: Object.values(messagesData || {}),
-                      backgroundColor: '#7D54D1',
-                    },
-                  ],
-                }}
-                legendID="legend-container-1"
-                maxTicksLimitInX={8}
-                isStacked={false}
-              />
+              {loadingMessages ? (
+                <SkeletonLoader count={1} component={'analysis'} />
+              ) : (
+                <AnalyticsChart
+                  data={{
+                    labels: Object.keys(messagesData || {}).map((item) =>
+                      formatDate(item),
+                    ),
+                    datasets: [
+                      {
+                        label: 'Messages',
+                        data: Object.values(messagesData || {}),
+                        backgroundColor: '#7D54D1',
+                      },
+                    ],
+                  }}
+                  legendID="legend-container-1"
+                  maxTicksLimitInX={8}
+                  isStacked={false}
+                />
+              )}
             </div>
           </div>
-          <div className="h-[345px] [@media(min-width:1080px)]:w-1/2 w-full px-6 py-5 border rounded-2xl border-silver dark:border-silver/40">
+
+          {/* Token Usage Analytics */}
+          <div className="h-[345px] [@media(min-width:1080px)]:w-1/2 w-full px-6 py-5 border rounded-2xl border-silver dark:border-silver/40 overflow-hidden">
             <div className="flex flex-row items-center justify-start gap-3">
               <p className="font-bold text-jet dark:text-bright-gray">
                 Token Usage
@@ -253,31 +273,37 @@ export default function Analytics() {
                 id="legend-container-2"
                 className="flex flex-row items-center justify-end"
               ></div>
-              <AnalyticsChart
-                data={{
-                  labels: Object.keys(tokenUsageData || {}).map((item) =>
-                    formatDate(item),
-                  ),
-                  datasets: [
-                    {
-                      label: 'Tokens',
-                      data: Object.values(tokenUsageData || {}),
-                      backgroundColor: '#7D54D1',
-                    },
-                  ],
-                }}
-                legendID="legend-container-2"
-                maxTicksLimitInX={8}
-                isStacked={false}
-              />
+              {loadingTokens ? (
+                <SkeletonLoader count={1} component={'analysis'} />
+              ) : (
+                <AnalyticsChart
+                  data={{
+                    labels: Object.keys(tokenUsageData || {}).map((item) =>
+                      formatDate(item),
+                    ),
+                    datasets: [
+                      {
+                        label: 'Tokens',
+                        data: Object.values(tokenUsageData || {}),
+                        backgroundColor: '#7D54D1',
+                      },
+                    ],
+                  }}
+                  legendID="legend-container-2"
+                  maxTicksLimitInX={8}
+                  isStacked={false}
+                />
+              )}
             </div>
           </div>
         </div>
-        <div className="mt-8 w-full">
-          <div className="h-[345px] w-full px-6 py-5 border rounded-2xl border-silver dark:border-silver/40">
+
+        {/* Feedback Analytics */}
+        <div className="mt-8 w-full flex flex-col gap-3">
+          <div className="h-[345px] w-full px-6 py-5 border rounded-2xl border-silver dark:border-silver/40 overflow-hidden">
             <div className="flex flex-row items-center justify-start gap-3">
               <p className="font-bold text-jet dark:text-bright-gray">
-                User Feedback
+                Feedback
               </p>
               <Dropdown
                 size="w-[125px]"
@@ -300,32 +326,36 @@ export default function Analytics() {
                 id="legend-container-3"
                 className="flex flex-row items-center justify-end"
               ></div>
-              <AnalyticsChart
-                data={{
-                  labels: Object.keys(feedbackData || {}).map((item) =>
-                    formatDate(item),
-                  ),
-                  datasets: [
-                    {
-                      label: 'Positive',
-                      data: Object.values(feedbackData || {}).map(
-                        (item) => item.positive,
-                      ),
-                      backgroundColor: '#8BD154',
-                    },
-                    {
-                      label: 'Negative',
-                      data: Object.values(feedbackData || {}).map(
-                        (item) => item.negative,
-                      ),
-                      backgroundColor: '#D15454',
-                    },
-                  ],
-                }}
-                legendID="legend-container-3"
-                maxTicksLimitInX={10}
-                isStacked={true}
-              />
+              {loadingFeedback ? (
+                <SkeletonLoader count={1} component={'analysis'} />
+              ) : (
+                <AnalyticsChart
+                  data={{
+                    labels: Object.keys(feedbackData || {}).map((item) =>
+                      formatDate(item),
+                    ),
+                    datasets: [
+                      {
+                        label: 'Positive Feedback',
+                        data: Object.values(feedbackData || {}).map(
+                          (item) => item.positive,
+                        ),
+                        backgroundColor: '#7D54D1',
+                      },
+                      {
+                        label: 'Negative Feedback',
+                        data: Object.values(feedbackData || {}).map(
+                          (item) => item.negative,
+                        ),
+                        backgroundColor: '#FF6384',
+                      },
+                    ],
+                  }}
+                  legendID="legend-container-3"
+                  maxTicksLimitInX={8}
+                  isStacked={false}
+                />
+              )}
             </div>
           </div>
         </div>

frontend/src/settings/Documents.tsx

@@ -1,3 +1,4 @@
+import React, { useState } from 'react';
 import PropTypes from 'prop-types';
 import { useTranslation } from 'react-i18next';
 import { useDispatch } from 'react-redux';
@@ -5,10 +6,14 @@ import { useDispatch } from 'react-redux';
 import userService from '../api/services/userService';
 import SyncIcon from '../assets/sync.svg';
 import Trash from '../assets/trash.svg';
+import caretSort from '../assets/caret-sort.svg';
 import DropdownMenu from '../components/DropdownMenu';
-import { Doc, DocumentsProps } from '../models/misc';
+import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported
+import SkeletonLoader from '../components/SkeletonLoader';
 import { getDocs } from '../preferences/preferenceApi';
 import { setSourceDocs } from '../preferences/preferenceSlice';
+import Input from '../components/Input';
+import Upload from '../upload/Upload'; // Import the Upload component
 
 // Utility function to format numbers
 const formatTokens = (tokens: number): string => {
@@ -33,6 +38,14 @@ const Documents: React.FC<DocumentsProps> = ({
 }) => {
   const { t } = useTranslation();
   const dispatch = useDispatch();
+
+  // State for search input
+  const [searchTerm, setSearchTerm] = useState('');
+  // State for modal: active/inactive
+  const [modalState, setModalState] = useState<ActiveState>('INACTIVE'); // Initialize with inactive state
+  const [isOnboarding, setIsOnboarding] = useState(false); // State for onboarding flag
+  const [loading, setLoading] = useState(false);
+
   const syncOptions = [
     { label: 'Never', value: 'never' },
     { label: 'Daily', value: 'daily' },
@@ -41,6 +54,7 @@ const Documents: React.FC<DocumentsProps> = ({
   ];
 
   const handleManageSync = (doc: Doc, sync_frequency: string) => {
+    setLoading(true);
     userService
       .manageSync({ source_id: doc.id, sync_frequency })
       .then(() => {
@@ -49,81 +63,146 @@ const Documents: React.FC<DocumentsProps> = ({
       .then((data) => {
         dispatch(setSourceDocs(data));
       })
-      .catch((error) => console.error(error));
+      .catch((error) => console.error(error))
+      .finally(() => {
+        setLoading(false);
+      });
   };
+
+  // Filter documents based on the search term
+  const filteredDocuments = documents?.filter((document) =>
+    document.name.toLowerCase().includes(searchTerm.toLowerCase()),
+  );
+
   return (
     <div className="mt-8">
       <div className="flex flex-col relative">
         <div className="z-10 w-full overflow-x-auto">
-          <table className="table-default">
-            <thead>
-              <tr>
-                <th>{t('settings.documents.name')}</th>
-                <th>{t('settings.documents.date')}</th>
-                <th>{t('settings.documents.tokenUsage')}</th>
-                <th>{t('settings.documents.type')}</th>
-                <th></th>
-              </tr>
-            </thead>
-            <tbody>
-              {!documents?.length && (
+          <div className="my-3 flex justify-between items-center">
+            <div className="p-1">
+              <Input
+                maxLength={256}
+                placeholder="Search..."
+                name="Document-search-input"
+                type="text"
+                id="document-search-input"
+                value={searchTerm}
+                onChange={(e) => setSearchTerm(e.target.value)} // Handle search input change
+              />
+            </div>
+            <button
+              className="rounded-full w-40 bg-purple-30 px-4 py-3 text-white hover:bg-[#6F3FD1]"
+              onClick={() => {
+                setIsOnboarding(false); // Set onboarding flag if needed
+                setModalState('ACTIVE'); // Open the upload modal
+              }}
+            >
+              Add New
+            </button>
+          </div>
+          {loading ? (
+            <SkeletonLoader count={1} />
+          ) : (
+            <table className="table-default">
+              <thead>
                 <tr>
-                  <td colSpan={5} className="!p-4">
-                    {t('settings.documents.noData')}
-                  </td>
+                  <th>{t('settings.documents.name')}</th>
+                  <th>
+                    <div className="flex justify-center items-center">
+                      {t('settings.documents.date')}
+                      <img src={caretSort} alt="" />
+                    </div>
+                  </th>
+                  <th>
+                    <div className="flex justify-center items-center">
+                      {t('settings.documents.tokenUsage')}
+                      <img src={caretSort} alt="" />
+                    </div>
+                  </th>
+                  <th>
+                    <div className="flex justify-center items-center">
+                      {t('settings.documents.type')}
+                      <img src={caretSort} alt="" />
+                    </div>
+                  </th>
+                  <th></th>
                 </tr>
-              )}
-              {documents &&
-                documents.map((document, index) => (
-                  <tr key={index}>
-                    <td>{document.name}</td>
-                    <td>{document.date}</td>
-                    <td>
-                      {document.tokens ? formatTokens(+document.tokens) : ''}
-                    </td>
-                    <td>
-                      {document.type === 'remote' ? 'Pre-loaded' : 'Private'}
-                    </td>
-                    <td>
-                      <div className="flex flex-row items-center">
-                        {document.type !== 'remote' && (
-                          <img
-                            src={Trash}
-                            alt="Delete"
-                            className="h-4 w-4 cursor-pointer hover:opacity-50"
-                            id={`img-${index}`}
-                            onClick={(event) => {
-                              event.stopPropagation();
-                              handleDeleteDocument(index, document);
-                            }}
-                          />
-                        )}
-                        {document.syncFrequency && (
-                          <div className="ml-2">
-                            <DropdownMenu
-                              name="Sync"
-                              options={syncOptions}
-                              onSelect={(value: string) => {
-                                handleManageSync(document, value);
-                              }}
-                              defaultValue={document.syncFrequency}
-                              icon={SyncIcon}
-                            />
-                          </div>
-                        )}
-                      </div>
+              </thead>
+              <tbody>
+                {!filteredDocuments?.length && (
+                  <tr>
+                    <td colSpan={5} className="!p-4">
+                      {t('settings.documents.noData')}
                     </td>
                   </tr>
-                ))}
-            </tbody>
-          </table>
+                )}
+                {filteredDocuments &&
+                  filteredDocuments.map((document, index) => (
+                    <tr key={index}>
+                      <td>{document.name}</td>
+                      <td>{document.date}</td>
+                      <td>
+                        {document.tokens ? formatTokens(+document.tokens) : ''}
+                      </td>
+                      <td>
+                        {document.type === 'remote' ? 'Pre-loaded' : 'Private'}
+                      </td>
+                      <td>
+                        <div className="flex flex-row items-center">
+                          {document.type !== 'remote' && (
+                            <img
+                              src={Trash}
+                              alt="Delete"
+                              className="h-4 w-4 cursor-pointer opacity-60 hover:opacity-100"
+                              id={`img-${index}`}
+                              onClick={(event) => {
+                                event.stopPropagation();
+                                handleDeleteDocument(index, document);
+                              }}
+                            />
+                          )}
+                          {document.syncFrequency && (
+                            <div className="ml-2">
+                              <DropdownMenu
+                                name="Sync"
+                                options={syncOptions}
+                                onSelect={(value: string) => {
+                                  handleManageSync(document, value);
+                                }}
+                                defaultValue={document.syncFrequency}
+                                icon={SyncIcon}
+                              />
+                            </div>
+                          )}
+                        </div>
+                      </td>
+                    </tr>
+                  ))}
+              </tbody>
+            </table>
+          )}
         </div>
+        {/* Conditionally render the Upload modal based on modalState */}
+        {modalState === 'ACTIVE' && (
+          <div className="fixed top-0 left-0 w-screen h-screen z-50 flex items-center justify-center bg-transparent">
+            <div className="w-full h-full bg-transparent flex flex-col items-center justify-center p-8">
+              {/* Your Upload component */}
+              <Upload
+                modalState={modalState}
+                setModalState={setModalState}
+                isOnboarding={isOnboarding}
+              />
+            </div>
+          </div>
+        )}
       </div>
     </div>
   );
 };
+
 Documents.propTypes = {
   documents: PropTypes.array.isRequired,
   handleDeleteDocument: PropTypes.func.isRequired,
 };
+
 export default Documents;

frontend/src/settings/Logs.tsx

@@ -1,20 +1,23 @@
-import React from 'react';
+import React, { useState, useEffect, useRef, useCallback } from 'react';
 
 import userService from '../api/services/userService';
 import ChevronRight from '../assets/chevron-right.svg';
 import Dropdown from '../components/Dropdown';
+import SkeletonLoader from '../components/SkeletonLoader';
 import { APIKeyData, LogData } from './types';
 import CoppyButton from '../components/CopyButton';
 
 export default function Logs() {
-  const [chatbots, setChatbots] = React.useState<APIKeyData[]>([]);
-  const [selectedChatbot, setSelectedChatbot] =
-    React.useState<APIKeyData | null>();
-  const [logs, setLogs] = React.useState<LogData[]>([]);
-  const [page, setPage] = React.useState(1);
-  const [hasMore, setHasMore] = React.useState(true);
+  const [chatbots, setChatbots] = useState<APIKeyData[]>([]);
+  const [selectedChatbot, setSelectedChatbot] = useState<APIKeyData | null>();
+  const [logs, setLogs] = useState<LogData[]>([]);
+  const [page, setPage] = useState(1);
+  const [hasMore, setHasMore] = useState(true);
+  const [loadingChatbots, setLoadingChatbots] = useState(true);
+  const [loadingLogs, setLoadingLogs] = useState(true);
 
   const fetchChatbots = async () => {
+    setLoadingChatbots(true);
     try {
       const response = await userService.getAPIKeys();
       if (!response.ok) {
@@ -24,10 +27,13 @@ export default function Logs() {
       setChatbots(chatbots);
     } catch (error) {
       console.error(error);
+    } finally {
+      setLoadingChatbots(false);
     }
   };
 
   const fetchLogs = async () => {
+    setLoadingLogs(true);
     try {
       const response = await userService.getLogs({
         page: page,
@@ -38,20 +44,23 @@ export default function Logs() {
         throw new Error('Failed to fetch logs');
       }
       const olderLogs = await response.json();
-      setLogs([...logs, ...olderLogs.logs]);
+      setLogs((prevLogs) => [...prevLogs, ...olderLogs.logs]);
       setHasMore(olderLogs.has_more);
     } catch (error) {
       console.error(error);
+    } finally {
+      setLoadingLogs(false);
     }
   };
 
-  React.useEffect(() => {
+  useEffect(() => {
     fetchChatbots();
   }, []);
 
-  React.useEffect(() => {
+  useEffect(() => {
     if (hasMore) fetchLogs();
   }, [page, selectedChatbot]);
+
   return (
     <div className="mt-12">
       <div className="flex flex-col items-start">
@@ -59,38 +68,47 @@ export default function Logs() {
           <p className="font-bold text-jet dark:text-bright-gray">
             Filter by chatbot
           </p>
-          <Dropdown
-            size="w-[55vw] sm:w-[360px]"
-            options={[
-              ...chatbots.map((chatbot) => ({
-                label: chatbot.name,
-                value: chatbot.id,
-              })),
-              { label: 'None', value: '' },
-            ]}
-            placeholder="Select chatbot"
-            onSelect={(chatbot: { label: string; value: string }) => {
-              setSelectedChatbot(
-                chatbots.find((item) => item.id === chatbot.value),
-              );
-              setLogs([]);
-              setPage(1);
-              setHasMore(true);
-            }}
-            selectedValue={
-              (selectedChatbot && {
-                label: selectedChatbot.name,
-                value: selectedChatbot.id,
-              }) ||
-              null
-            }
-            rounded="3xl"
-            border="border"
-          />
+          {loadingChatbots ? (
+            <SkeletonLoader />
+          ) : (
+            <Dropdown
+              size="w-[55vw] sm:w-[360px]"
+              options={[
+                ...chatbots.map((chatbot) => ({
+                  label: chatbot.name,
+                  value: chatbot.id,
+                })),
+                { label: 'None', value: '' },
+              ]}
+              placeholder="Select chatbot"
+              onSelect={(chatbot: { label: string; value: string }) => {
+                setSelectedChatbot(
+                  chatbots.find((item) => item.id === chatbot.value),
+                );
+                setLogs([]);
+                setPage(1);
+                setHasMore(true);
+              }}
+              selectedValue={
+                (selectedChatbot && {
+                  label: selectedChatbot.name,
+                  value: selectedChatbot.id,
+                }) ||
+                null
+              }
+              rounded="3xl"
+              border="border"
+            />
+          )}
         </div>
       </div>
+
       <div className="mt-8">
-        <LogsTable logs={logs} setPage={setPage} />
+        {loadingLogs ? (
+          <SkeletonLoader component={'logs'} />
+        ) : (
+          <LogsTable logs={logs} setPage={setPage} />
+        )}
       </div>
     </div>
   );
@@ -102,15 +120,16 @@ type LogsTableProps = {
 };
 
 function LogsTable({ logs, setPage }: LogsTableProps) {
-  const observerRef = React.useRef<any>();
-  const firstObserver = React.useCallback((node: HTMLDivElement) => {
+  const observerRef = useRef<any>();
+  const firstObserver = useCallback((node: HTMLDivElement) => {
     if (observerRef.current) {
-      observerRef.current = new IntersectionObserver((enteries) => {
-        if (enteries[0].isIntersecting) setPage((prev) => prev + 1);
+      observerRef.current = new IntersectionObserver((entries) => {
+        if (entries[0].isIntersecting) setPage((prev) => prev + 1);
       });
     }
     if (node && observerRef.current) observerRef.current.observe(node);
   }, []);
+
   return (
     <div className="logs-table border rounded-2xl h-[55vh] w-full overflow-hidden border-silver dark:border-silver/40">
       <div className="h-8 bg-black/10 dark:bg-chinese-black flex flex-col items-start justify-center">

frontend/src/upload/Upload.tsx

@@ -314,7 +314,10 @@ function Upload({
       'application/zip': ['.zip'],
       'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
         ['.docx'],
+      'application/json': ['.json'],
       'text/csv': ['.csv'],
+      'text/html': ['.html'],
+      'application/epub+zip': ['.epub'],
       'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': [
         '.xlsx',
       ],

tests/llm/test_anthropic.py

@@ -22,17 +22,23 @@ class TestAnthropicLLM(unittest.TestCase):
         mock_response = Mock()
         mock_response.completion = "test completion"
 
-        with patch.object(self.llm.anthropic.completions, "create", return_value=mock_response) as mock_create:
-            response = self.llm.gen("test_model", messages)
-            self.assertEqual(response, "test completion")
+        with patch("application.cache.get_redis_instance") as mock_make_redis:
+            mock_redis_instance = mock_make_redis.return_value
+            mock_redis_instance.get.return_value = None
+            mock_redis_instance.set = Mock()
 
-            prompt_expected = "### Context \n context \n ### Question \n question"
-            mock_create.assert_called_with(
-                model="test_model",
-                max_tokens_to_sample=300,
-                stream=False,
-                prompt=f"{self.llm.HUMAN_PROMPT} {prompt_expected}{self.llm.AI_PROMPT}"
-            )
+            with patch.object(self.llm.anthropic.completions, "create", return_value=mock_response) as mock_create:
+                response = self.llm.gen("test_model", messages)
+                self.assertEqual(response, "test completion")
+
+                prompt_expected = "### Context \n context \n ### Question \n question"
+                mock_create.assert_called_with(
+                    model="test_model",
+                    max_tokens_to_sample=300,
+                    stream=False,
+                    prompt=f"{self.llm.HUMAN_PROMPT} {prompt_expected}{self.llm.AI_PROMPT}"
+                )
+            mock_redis_instance.set.assert_called_once()
 
     def test_gen_stream(self):
         messages = [
@@ -41,17 +47,23 @@ class TestAnthropicLLM(unittest.TestCase):
         ]
         mock_responses = [Mock(completion="response_1"), Mock(completion="response_2")]
 
-        with patch.object(self.llm.anthropic.completions, "create", return_value=iter(mock_responses)) as mock_create:
-            responses = list(self.llm.gen_stream("test_model", messages))
-            self.assertListEqual(responses, ["response_1", "response_2"])
+        with patch("application.cache.get_redis_instance") as mock_make_redis:
+            mock_redis_instance = mock_make_redis.return_value
+            mock_redis_instance.get.return_value = None
+            mock_redis_instance.set = Mock()
 
-            prompt_expected = "### Context \n context \n ### Question \n question"
-            mock_create.assert_called_with(
-                model="test_model",
-                prompt=f"{self.llm.HUMAN_PROMPT} {prompt_expected}{self.llm.AI_PROMPT}",
-                max_tokens_to_sample=300,
-                stream=True
-            )
+            with patch.object(self.llm.anthropic.completions, "create", return_value=iter(mock_responses)) as mock_create:
+                responses = list(self.llm.gen_stream("test_model", messages))
+                self.assertListEqual(responses, ["response_1", "response_2"])
+
+                prompt_expected = "### Context \n context \n ### Question \n question"
+                mock_create.assert_called_with(
+                    model="test_model",
+                    prompt=f"{self.llm.HUMAN_PROMPT} {prompt_expected}{self.llm.AI_PROMPT}",
+                    max_tokens_to_sample=300,
+                    stream=True
+                )
+            mock_redis_instance.set.assert_called_once()
 
 if __name__ == "__main__":
     unittest.main()

tests/llm/test_sagemaker.py

@@ -52,28 +52,38 @@ class TestSagemakerAPILLM(unittest.TestCase):
         self.response['Body'].read.return_value.decode.return_value = json.dumps(self.result)
         
     def test_gen(self):
-        with patch.object(self.sagemaker.runtime, 'invoke_endpoint', 
-                          return_value=self.response) as mock_invoke_endpoint:
-            output = self.sagemaker.gen(None, self.messages)
-            mock_invoke_endpoint.assert_called_once_with(
-                EndpointName=self.sagemaker.endpoint,
-                ContentType='application/json',
-                Body=self.body_bytes
-            )
-            self.assertEqual(output, 
-                             self.result[0]['generated_text'][len(self.prompt):])
+        with patch('application.cache.get_redis_instance') as mock_make_redis:
+            mock_redis_instance = mock_make_redis.return_value
+            mock_redis_instance.get.return_value = None
+
+            with patch.object(self.sagemaker.runtime, 'invoke_endpoint', 
+                            return_value=self.response) as mock_invoke_endpoint:
+                output = self.sagemaker.gen(None, self.messages)
+                mock_invoke_endpoint.assert_called_once_with(
+                    EndpointName=self.sagemaker.endpoint,
+                    ContentType='application/json',
+                    Body=self.body_bytes
+                )
+                self.assertEqual(output, 
+                                self.result[0]['generated_text'][len(self.prompt):])
+            mock_make_redis.assert_called_once()
+            mock_redis_instance.set.assert_called_once()
     
     def test_gen_stream(self):
-        with patch.object(self.sagemaker.runtime, 'invoke_endpoint_with_response_stream', 
-                          return_value=self.response) as mock_invoke_endpoint:
-            output = list(self.sagemaker.gen_stream(None, self.messages))
-            mock_invoke_endpoint.assert_called_once_with(
-                EndpointName=self.sagemaker.endpoint,
-                ContentType='application/json',
-                Body=self.body_bytes_stream
-            )
-            self.assertEqual(output, [])
-            
+        with patch('application.cache.get_redis_instance') as mock_make_redis:
+            mock_redis_instance = mock_make_redis.return_value
+            mock_redis_instance.get.return_value = None
+
+            with patch.object(self.sagemaker.runtime, 'invoke_endpoint_with_response_stream', 
+                            return_value=self.response) as mock_invoke_endpoint:
+                output = list(self.sagemaker.gen_stream(None, self.messages))
+                mock_invoke_endpoint.assert_called_once_with(
+                    EndpointName=self.sagemaker.endpoint,
+                    ContentType='application/json',
+                    Body=self.body_bytes_stream
+                )
+                self.assertEqual(output, [])
+            mock_redis_instance.set.assert_called_once()
 class TestLineIterator(unittest.TestCase):
     
     def setUp(self):

tests/test_cache.py

@@ -0,0 +1,131 @@
+import unittest
+import json
+from unittest.mock import patch, MagicMock
+from application.cache import gen_cache_key, stream_cache, gen_cache
+from application.utils import get_hash
+
+
+# Test for gen_cache_key function
+def test_make_gen_cache_key():
+    messages = [
+        {'role': 'user', 'content': 'test_user_message'},
+        {'role': 'system', 'content': 'test_system_message'},
+    ]
+    model = "test_docgpt"
+    
+    # Manually calculate the expected hash
+    expected_combined = f"{model}_{json.dumps(messages, sort_keys=True)}"
+    expected_hash = get_hash(expected_combined)
+    cache_key = gen_cache_key(*messages, model=model)
+
+    assert cache_key == expected_hash
+
+def test_gen_cache_key_invalid_message_format():
+    # Test when messages is not a list
+    with unittest.TestCase.assertRaises(unittest.TestCase, ValueError) as context:
+        gen_cache_key("This is not a list", model="docgpt")
+    assert str(context.exception) == "All messages must be dictionaries."
+
+# Test for gen_cache decorator
+@patch('application.cache.get_redis_instance')  # Mock the Redis client
+def test_gen_cache_hit(mock_make_redis):
+    # Arrange
+    mock_redis_instance = MagicMock()
+    mock_make_redis.return_value = mock_redis_instance
+    mock_redis_instance.get.return_value = b"cached_result"  # Simulate a cache hit
+
+    @gen_cache
+    def mock_function(self, model, messages):
+        return "new_result"
+
+    messages = [{'role': 'user', 'content': 'test_user_message'}]
+    model = "test_docgpt"
+
+    # Act
+    result = mock_function(None, model, messages)
+
+    # Assert
+    assert result == "cached_result"  # Should return cached result
+    mock_redis_instance.get.assert_called_once()  # Ensure Redis get was called
+    mock_redis_instance.set.assert_not_called()  # Ensure the function result is not cached again
+
+
+@patch('application.cache.get_redis_instance')  # Mock the Redis client
+def test_gen_cache_miss(mock_make_redis):
+    # Arrange
+    mock_redis_instance = MagicMock()
+    mock_make_redis.return_value = mock_redis_instance
+    mock_redis_instance.get.return_value = None  # Simulate a cache miss
+
+    @gen_cache
+    def mock_function(self, model, messages):
+        return "new_result"
+
+    messages = [
+        {'role': 'user', 'content': 'test_user_message'},
+        {'role': 'system', 'content': 'test_system_message'},
+    ]
+    model = "test_docgpt"
+    # Act
+    result = mock_function(None, model, messages)
+
+    # Assert
+    assert result == "new_result"
+    mock_redis_instance.get.assert_called_once() 
+
+@patch('application.cache.get_redis_instance')  
+def test_stream_cache_hit(mock_make_redis):
+    # Arrange
+    mock_redis_instance = MagicMock()
+    mock_make_redis.return_value = mock_redis_instance
+
+    cached_chunk = json.dumps(["chunk1", "chunk2"]).encode('utf-8')
+    mock_redis_instance.get.return_value = cached_chunk
+
+    @stream_cache
+    def mock_function(self, model, messages, stream):
+        yield "new_chunk"
+
+    messages = [{'role': 'user', 'content': 'test_user_message'}]
+    model = "test_docgpt"
+
+    # Act
+    result = list(mock_function(None, model, messages, stream=True))
+
+    # Assert
+    assert result == ["chunk1", "chunk2"]  # Should return cached chunks
+    mock_redis_instance.get.assert_called_once()
+    mock_redis_instance.set.assert_not_called()
+
+
+@patch('application.cache.get_redis_instance')
+def test_stream_cache_miss(mock_make_redis):
+    # Arrange
+    mock_redis_instance = MagicMock()
+    mock_make_redis.return_value = mock_redis_instance
+    mock_redis_instance.get.return_value = None  # Simulate a cache miss
+
+    @stream_cache
+    def mock_function(self, model, messages, stream):
+        yield "new_chunk"
+
+    messages = [
+        {'role': 'user', 'content': 'This is the context'},
+        {'role': 'system', 'content': 'Some other message'},
+        {'role': 'user', 'content': 'What is the answer?'}
+    ]
+    model = "test_docgpt"
+
+    # Act
+    result = list(mock_function(None, model, messages, stream=True))
+
+    # Assert
+    assert result == ["new_chunk"]
+    mock_redis_instance.get.assert_called_once()  
+    mock_redis_instance.set.assert_called_once()  
+
+
+
+    
+
+