Merge branch 'dependabot/npm_and_yarn/frontend/reduxjs/toolkit-2.2.7' of https://github.com/arc53/docsgpt into dependabot/npm_and_yarn/frontend/reduxjs/toolkit-2.2.7

application/api/answer/routes.py

@@ -1,7 +1,7 @@
 import asyncio
 import os
 import sys
-from flask import Blueprint, request, Response
+from flask import Blueprint, request, Response, current_app
 import json
 import datetime
 import logging
@@ -9,6 +9,7 @@ import traceback
 
 from pymongo import MongoClient
 from bson.objectid import ObjectId
+from bson.dbref import DBRef
 
 from application.core.settings import settings
 from application.llm.llm_creator import LLMCreator
@@ -20,9 +21,10 @@ logger = logging.getLogger(__name__)
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 prompts_collection = db["prompts"]
 api_key_collection = db["api_keys"]
+user_logs_collection = db["user_logs"]
 answer = Blueprint("answer", __name__)
 
 gpt_model = ""
@@ -74,27 +76,29 @@ def run_async_chain(chain, question, chat_history):
 
 def get_data_from_api_key(api_key):
     data = api_key_collection.find_one({"key": api_key})
-
     # # Raise custom exception if the API key is not found
     if data is None:
         raise Exception("Invalid API Key, please generate new key", 401)
+
+    if "retriever" not in data:
+        data["retriever"] = None
+
+    if "source" in data and isinstance(data["source"], DBRef):
+        source_doc = db.dereference(data["source"])
+        data["source"] = str(source_doc["_id"])
+        if "retriever" in source_doc:
+            data["retriever"] = source_doc["retriever"]
+    else:
+        data["source"] = {}
     return data
 
 
-def get_vectorstore(data):
-    if "active_docs" in data:
-        if data["active_docs"].split("/")[0] == "default":
-            vectorstore = ""
-        elif data["active_docs"].split("/")[0] == "local":
-            vectorstore = "indexes/" + data["active_docs"]
-        else:
-            vectorstore = "vectors/" + data["active_docs"]
-        if data["active_docs"] == "default":
-            vectorstore = ""
-    else:
-        vectorstore = ""
-    vectorstore = os.path.join("application", vectorstore)
-    return vectorstore
+def get_retriever(source_id: str):
+    doc = sources_collection.find_one({"_id": ObjectId(source_id)})
+    if doc is None:
+        raise Exception("Source document does not exist", 404)
+    retriever_name = None if "retriever" not in doc else doc["retriever"]
+    return retriever_name
 
 
 def is_azure_configured():
@@ -203,6 +207,21 @@ def complete_stream(
             data = json.dumps({"type": "id", "id": str(conversation_id)})
             yield f"data: {data}\n\n"
 
+        retriever_params = retriever.get_params()
+        user_logs_collection.insert_one(
+            {
+                "action": "stream_answer",
+                "level": "info",
+                "user": "local",
+                "api_key": user_api_key,
+                "question": question,
+                "response": response_full,
+                "sources": source_log_docs,
+                "retriever_params": retriever_params,
+                "timestamp": datetime.datetime.now(datetime.timezone.utc),
+            }
+        )
+
         data = json.dumps({"type": "end"})
         yield f"data: {data}\n\n"
     except Exception as e:
@@ -247,25 +266,31 @@ def stream():
         else:
             token_limit = settings.DEFAULT_MAX_HISTORY
 
-        # check if active_docs or api_key is set
+        ## retriever can be "brave_search, duckduck_search or classic"
+        retriever_name = data["retriever"] if "retriever" in data else "classic"
 
+        # check if active_docs or api_key is set
         if "api_key" in data:
             data_key = get_data_from_api_key(data["api_key"])
             chunks = int(data_key["chunks"])
             prompt_id = data_key["prompt_id"]
             source = {"active_docs": data_key["source"]}
+            retriever_name = data_key["retriever"] or retriever_name
             user_api_key = data["api_key"]
+
         elif "active_docs" in data:
             source = {"active_docs": data["active_docs"]}
+            retriever_name = get_retriever(data["active_docs"]) or retriever_name
             user_api_key = None
+
         else:
             source = {}
             user_api_key = None
 
-        if source["active_docs"].split("/")[0] in ["default", "local"]:
-            retriever_name = "classic"
-        else:
-            retriever_name = source["active_docs"]
+        current_app.logger.info(
+            f"/stream - request_data: {data}, source: {source}",
+            extra={"data": json.dumps({"request_data": data, "source": source})},
+        )
 
         prompt = get_prompt(prompt_id)
 
@@ -301,7 +326,10 @@ def stream():
             mimetype="text/event-stream",
         )
     except Exception as e:
-        print("\033[91merr", str(e), file=sys.stderr)
+        current_app.logger.error(
+            f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
+            extra={"error": str(e), "traceback": traceback.format_exc()},
+        )
         message = e.args[0]
         status_code = 400
         # # Custom exceptions with two arguments, index 1 as status code
@@ -345,6 +373,9 @@ def api_answer():
     else:
         token_limit = settings.DEFAULT_MAX_HISTORY
 
+    ## retriever can be brave_search, duckduck_search or classic
+    retriever_name = data["retriever"] if "retriever" in data else "classic"
+
     # use try and except  to check for exception
     try:
         # check if the vectorstore is set
@@ -353,18 +384,23 @@ def api_answer():
             chunks = int(data_key["chunks"])
             prompt_id = data_key["prompt_id"]
             source = {"active_docs": data_key["source"]}
+            retriever_name = data_key["retriever"] or retriever_name
             user_api_key = data["api_key"]
+        elif "active_docs" in data:
+            source = {"active_docs": data["active_docs"]}
+            retriever_name = get_retriever(data["active_docs"]) or retriever_name
+            user_api_key = None
         else:
-            source = data
+            source = {}
             user_api_key = None
 
-        if source["active_docs"].split("/")[0] in ["default", "local"]:
-            retriever_name = "classic"
-        else:
-            retriever_name = source["active_docs"]
-
         prompt = get_prompt(prompt_id)
 
+        current_app.logger.info(
+            f"/api/answer - request_data: {data}, source: {source}",
+            extra={"data": json.dumps({"request_data": data, "source": source})},
+        )
+
         retriever = RetrieverCreator.create_retriever(
             retriever_name,
             question=question,
@@ -393,15 +429,32 @@ def api_answer():
         )
 
         result = {"answer": response_full, "sources": source_log_docs}
-        result["conversation_id"] = save_conversation(
-            conversation_id, question, response_full, source_log_docs, llm
+        result["conversation_id"] = str(
+            save_conversation(
+                conversation_id, question, response_full, source_log_docs, llm
+            )
+        )
+        retriever_params = retriever.get_params()
+        user_logs_collection.insert_one(
+            {
+                "action": "api_answer",
+                "level": "info",
+                "user": "local",
+                "api_key": user_api_key,
+                "question": question,
+                "response": response_full,
+                "sources": source_log_docs,
+                "retriever_params": retriever_params,
+                "timestamp": datetime.datetime.now(datetime.timezone.utc),
+            }
         )
 
         return result
     except Exception as e:
-        # print whole traceback
-        traceback.print_exc()
-        print(str(e))
+        current_app.logger.error(
+            f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",
+            extra={"error": str(e), "traceback": traceback.format_exc()},
+        )
         return bad_request(500, str(e))
 
 
@@ -416,7 +469,7 @@ def api_search():
     if "api_key" in data:
         data_key = get_data_from_api_key(data["api_key"])
         chunks = int(data_key["chunks"])
-        source = {"active_docs": data_key["source"]}
+        source = {"active_docs":data_key["source"]}
         user_api_key = data["api_key"]
     elif "active_docs" in data:
         source = {"active_docs": data["active_docs"]}
@@ -425,15 +478,20 @@ def api_search():
         source = {}
         user_api_key = None
 
-    if source["active_docs"].split("/")[0] in ["default", "local"]:
-        retriever_name = "classic"
+    if "retriever" in data:
+        retriever_name = data["retriever"]
     else:
-        retriever_name = source["active_docs"]
+        retriever_name = "classic"
     if "token_limit" in data:
         token_limit = data["token_limit"]
     else:
         token_limit = settings.DEFAULT_MAX_HISTORY
 
+    current_app.logger.info(
+        f"/api/answer - request_data: {data}, source: {source}",
+        extra={"data": json.dumps({"request_data": data, "source": source})},
+    )
+
     retriever = RetrieverCreator.create_retriever(
         retriever_name,
         question=question,
@@ -447,6 +505,20 @@ def api_search():
     )
     docs = retriever.search()
 
+    retriever_params = retriever.get_params()
+    user_logs_collection.insert_one(
+        {
+            "action": "api_search",
+            "level": "info",
+            "user": "local",
+            "api_key": user_api_key,
+            "question": question,
+            "sources": docs,
+            "retriever_params": retriever_params,
+            "timestamp": datetime.datetime.now(datetime.timezone.utc),
+        }
+    )
+
     if data.get("isNoneDoc"):
         for doc in docs:
             doc["source"] = "None"

application/api/internal/routes.py

@@ -3,13 +3,13 @@ import datetime
 from flask import Blueprint, request, send_from_directory
 from pymongo import MongoClient
 from werkzeug.utils import secure_filename
-
+from bson.objectid import ObjectId
 
 from application.core.settings import settings
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 
 current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
@@ -35,7 +35,12 @@ def upload_index_files():
         return {"status": "no name"}
     job_name = secure_filename(request.form["name"])
     tokens = secure_filename(request.form["tokens"])
-    save_dir = os.path.join(current_dir, "indexes", user, job_name)
+    retriever = secure_filename(request.form["retriever"])
+    id = secure_filename(request.form["id"])
+    type = secure_filename(request.form["type"])
+    remote_data = secure_filename(request.form["remote_data"]) if "remote_data" in  request.form else None
+
+    save_dir = os.path.join(current_dir, "indexes", str(id))
     if settings.VECTOR_STORE == "faiss":
         if "file_faiss" not in request.files:
             print("No file part")
@@ -55,17 +60,19 @@ def upload_index_files():
             os.makedirs(save_dir)
         file_faiss.save(os.path.join(save_dir, "index.faiss"))
         file_pkl.save(os.path.join(save_dir, "index.pkl"))
-    # create entry in vectors_collection
-    vectors_collection.insert_one(
+    # create entry in sources_collection
+    sources_collection.insert_one(
         {
+            "_id": ObjectId(id),
             "user": user,
             "name": job_name,
             "language": job_name,
-            "location": save_dir,
             "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
             "model": settings.EMBEDDINGS_NAME,
-            "type": "local",
-            "tokens": tokens
+            "type": type,
+            "tokens": tokens,
+            "retriever": retriever,
+            "remote_data": remote_data
         }
     )
     return {"status": "ok"}

application/api/user/routes.py

@@ -1,14 +1,15 @@
+import datetime
 import os
-import uuid
 import shutil
-from flask import Blueprint, request, jsonify
-from urllib.parse import urlparse
-import requests
-from pymongo import MongoClient
-from bson.objectid import ObjectId
+import uuid
+
 from bson.binary import Binary, UuidRepresentation
-from werkzeug.utils import secure_filename
 from bson.dbref import DBRef
+from bson.objectid import ObjectId
+from flask import Blueprint, jsonify, request
+from pymongo import MongoClient
+from werkzeug.utils import secure_filename
+
 from application.api.user.tasks import ingest, ingest_remote
 
 from application.core.settings import settings
@@ -17,11 +18,13 @@ from application.vectorstore.vector_creator import VectorCreator
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 prompts_collection = db["prompts"]
 feedback_collection = db["feedback"]
 api_key_collection = db["api_keys"]
+token_usage_collection = db["token_usage"]
 shared_conversations_collections = db["shared_conversations"]
+user_logs_collection = db["user_logs"]
 
 user = Blueprint("user", __name__)
 
@@ -30,6 +33,27 @@ current_dir = os.path.dirname(
 )
 
 
+def generate_minute_range(start_date, end_date):
+    return {
+        (start_date + datetime.timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:00"): 0
+        for i in range(int((end_date - start_date).total_seconds() // 60) + 1)
+    }
+
+
+def generate_hourly_range(start_date, end_date):
+    return {
+        (start_date + datetime.timedelta(hours=i)).strftime("%Y-%m-%d %H:00"): 0
+        for i in range(int((end_date - start_date).total_seconds() // 3600) + 1)
+    }
+
+
+def generate_date_range(start_date, end_date):
+    return {
+        (start_date + datetime.timedelta(days=i)).strftime("%Y-%m-%d"): 0
+        for i in range((end_date - start_date).days + 1)
+    }
+
+
 @user.route("/api/delete_conversation", methods=["POST"])
 def delete_conversation():
     # deletes a conversation from the database
@@ -90,14 +114,15 @@ def api_feedback():
     question = data["question"]
     answer = data["answer"]
     feedback = data["feedback"]
-
-    feedback_collection.insert_one(
-        {
-            "question": question,
-            "answer": answer,
-            "feedback": feedback,
-        }
-    )
+    new_doc = {
+        "question": question,
+        "answer": answer,
+        "feedback": feedback,
+        "timestamp": datetime.datetime.now(datetime.timezone.utc),
+    }
+    if "api_key" in data:
+        new_doc["api_key"] = data["api_key"]
+    feedback_collection.insert_one(new_doc)
     return {"status": "ok"}
 
 
@@ -110,7 +135,7 @@ def delete_by_ids():
         return {"status": "error"}
 
     if settings.VECTOR_STORE == "faiss":
-        result = vectors_collection.delete_index(ids=ids)
+        result = sources_collection.delete_index(ids=ids)
         if result:
             return {"status": "ok"}
     return {"status": "error"}
@@ -121,27 +146,30 @@ def delete_old():
     """Delete old indexes."""
     import shutil
 
-    path = request.args.get("path")
-    dirs = path.split("/")
-    dirs_clean = []
-    for i in range(0, len(dirs)):
-        dirs_clean.append(secure_filename(dirs[i]))
-    # check that path strats with indexes or vectors
-
-    if dirs_clean[0] not in ["indexes", "vectors"]:
-        return {"status": "error"}
-    path_clean = "/".join(dirs_clean)
-    vectors_collection.delete_one({"name": dirs_clean[-1], "user": dirs_clean[-2]})
+    source_id = request.args.get("source_id")
+    doc = sources_collection.find_one(
+        {
+            "_id": ObjectId(source_id),
+            "user": "local",
+        }
+    )
+    if doc is None:
+        return {"status": "not found"}, 404
     if settings.VECTOR_STORE == "faiss":
         try:
-            shutil.rmtree(os.path.join(current_dir, path_clean))
+            shutil.rmtree(os.path.join(current_dir, str(doc["_id"])))
         except FileNotFoundError:
             pass
     else:
         vetorstore = VectorCreator.create_vectorstore(
-            settings.VECTOR_STORE, path=os.path.join(current_dir, path_clean)
+            settings.VECTOR_STORE, source_id=str(doc["_id"])
         )
         vetorstore.delete_index()
+    sources_collection.delete_one(
+        {
+            "_id": ObjectId(source_id),
+        }
+    )
 
     return {"status": "ok"}
 
@@ -248,54 +276,38 @@ def combined_json():
     data = [
         {
             "name": "default",
-            "language": "default",
-            "version": "",
-            "description": "default",
-            "fullName": "default",
             "date": "default",
-            "docLink": "default",
             "model": settings.EMBEDDINGS_NAME,
             "location": "remote",
             "tokens": "",
+            "retriever": "classic",
         }
     ]
     # structure: name, language, version, description, fullName, date, docLink
-    # append data from vectors_collection in sorted order in descending order of date
-    for index in vectors_collection.find({"user": user}).sort("date", -1):
+    # append data from sources_collection in sorted order in descending order of date
+    for index in sources_collection.find({"user": user}).sort("date", -1):
         data.append(
             {
+                "id": str(index["_id"]),
                 "name": index["name"],
-                "language": index["language"],
-                "version": "",
-                "description": index["name"],
-                "fullName": index["name"],
                 "date": index["date"],
-                "docLink": index["location"],
                 "model": settings.EMBEDDINGS_NAME,
                 "location": "local",
                 "tokens": index["tokens"] if ("tokens" in index.keys()) else "",
+                "retriever": (
+                    index["retriever"] if ("retriever" in index.keys()) else "classic"
+                ),
             }
         )
-    if settings.VECTOR_STORE == "faiss":
-        data_remote = requests.get(
-            "https://d3dg1063dc54p9.cloudfront.net/combined.json"
-        ).json()
-        for index in data_remote:
-            index["location"] = "remote"
-            data.append(index)
     if "duckduck_search" in settings.RETRIEVERS_ENABLED:
         data.append(
             {
                 "name": "DuckDuckGo Search",
-                "language": "en",
-                "version": "",
-                "description": "duckduck_search",
-                "fullName": "DuckDuckGo Search",
                 "date": "duckduck_search",
-                "docLink": "duckduck_search",
                 "model": settings.EMBEDDINGS_NAME,
                 "location": "custom",
                 "tokens": "",
+                "retriever": "duckduck_search",
             }
         )
     if "brave_search" in settings.RETRIEVERS_ENABLED:
@@ -303,14 +315,11 @@ def combined_json():
             {
                 "name": "Brave Search",
                 "language": "en",
-                "version": "",
-                "description": "brave_search",
-                "fullName": "Brave Search",
                 "date": "brave_search",
-                "docLink": "brave_search",
                 "model": settings.EMBEDDINGS_NAME,
                 "location": "custom",
                 "tokens": "",
+                "retriever": "brave_search",
             }
         )
 
@@ -319,39 +328,13 @@ def combined_json():
 
 @user.route("/api/docs_check", methods=["POST"])
 def check_docs():
-    # check if docs exist in a vectorstore folder
     data = request.get_json()
-    # split docs on / and take first part
-    if data["docs"].split("/")[0] == "local":
-        return {"status": "exists"}
+
     vectorstore = "vectors/" + secure_filename(data["docs"])
-    base_path = "https://raw.githubusercontent.com/arc53/DocsHUB/main/"
     if os.path.exists(vectorstore) or data["docs"] == "default":
         return {"status": "exists"}
     else:
-        file_url = urlparse(base_path + vectorstore + "index.faiss")
-
-        if (
-            file_url.scheme in ["https"]
-            and file_url.netloc == "raw.githubusercontent.com"
-            and file_url.path.startswith("/arc53/DocsHUB/main/")
-        ):
-            r = requests.get(file_url.geturl())
-            if r.status_code != 200:
-                return {"status": "null"}
-            else:
-                if not os.path.exists(vectorstore):
-                    os.makedirs(vectorstore)
-                with open(vectorstore + "index.faiss", "wb") as f:
-                    f.write(r.content)
-
-                r = requests.get(base_path + vectorstore + "index.pkl")
-                with open(vectorstore + "index.pkl", "wb") as f:
-                    f.write(r.content)
-        else:
-            return {"status": "null"}
-
-        return {"status": "loaded"}
+        return {"status": "not found"}
 
 
 @user.route("/api/create_prompt", methods=["POST"])
@@ -448,12 +431,23 @@ def get_api_keys():
     keys = api_key_collection.find({"user": user})
     list_keys = []
     for key in keys:
+        if "source" in key and isinstance(key["source"], DBRef):
+            source = db.dereference(key["source"])
+            if source is None:
+                continue
+            else:
+                source_name = source["name"]
+        elif "retriever" in key:
+            source_name = key["retriever"]
+        else:
+            continue
+
         list_keys.append(
             {
                 "id": str(key["_id"]),
                 "name": key["name"],
                 "key": key["key"][:4] + "..." + key["key"][-4:],
-                "source": key["source"],
+                "source": source_name,
                 "prompt_id": key["prompt_id"],
                 "chunks": key["chunks"],
             }
@@ -465,21 +459,22 @@ def get_api_keys():
 def create_api_key():
     data = request.get_json()
     name = data["name"]
-    source = data["source"]
     prompt_id = data["prompt_id"]
     chunks = data["chunks"]
     key = str(uuid.uuid4())
     user = "local"
-    resp = api_key_collection.insert_one(
-        {
-            "name": name,
-            "key": key,
-            "source": source,
-            "user": user,
-            "prompt_id": prompt_id,
-            "chunks": chunks,
-        }
-    )
+    new_api_key = {
+        "name": name,
+        "key": key,
+        "user": user,
+        "prompt_id": prompt_id,
+        "chunks": chunks,
+    }
+    if "source" in data and ObjectId.is_valid(data["source"]):
+        new_api_key["source"] = DBRef("sources", ObjectId(data["source"]))
+    if "retriever" in data:
+        new_api_key["retriever"] = data["retriever"]
+    resp = api_key_collection.insert_one(new_api_key)
     new_id = str(resp.inserted_id)
     return {"id": new_id, "key": key}
 
@@ -509,26 +504,29 @@ def share_conversation():
         conversation = conversations_collection.find_one(
             {"_id": ObjectId(conversation_id)}
         )
+        if conversation is None:
+            raise Exception("Conversation does not exist")
         current_n_queries = len(conversation["queries"])
 
         ##generate binary representation of uuid
         explicit_binary = Binary.from_uuid(uuid.uuid4(), UuidRepresentation.STANDARD)
 
         if isPromptable:
-            source = "default" if "source" not in data else data["source"]
             prompt_id = "default" if "prompt_id" not in data else data["prompt_id"]
             chunks = "2" if "chunks" not in data else data["chunks"]
 
             name = conversation["name"] + "(shared)"
-            pre_existing_api_document = api_key_collection.find_one(
-                {
-                    "prompt_id": prompt_id,
-                    "chunks": chunks,
-                    "source": source,
-                    "user": user,
-                }
-            )
-            api_uuid = str(uuid.uuid4())
+            new_api_key_data = {
+                "prompt_id": prompt_id,
+                "chunks": chunks,
+                "user": user,
+            }
+            if "source" in data and ObjectId.is_valid(data["source"]):
+                new_api_key_data["source"] = DBRef("sources", ObjectId(data["source"]))
+            elif "retriever" in data:
+                new_api_key_data["retriever"] = data["retriever"]
+
+            pre_existing_api_document = api_key_collection.find_one(new_api_key_data)
             if pre_existing_api_document:
                 api_uuid = pre_existing_api_document["key"]
                 pre_existing = shared_conversations_collections.find_one(
@@ -570,29 +568,30 @@ def share_conversation():
                         {"success": True, "identifier": str(explicit_binary.as_uuid())}
                     )
             else:
-                api_key_collection.insert_one(
+
+                api_uuid = str(uuid.uuid4())
+                new_api_key_data["key"] = api_uuid
+                new_api_key_data["name"] = name
+                if "source" in data and ObjectId.is_valid(data["source"]):
+                    new_api_key_data["source"] = DBRef(
+                        "sources", ObjectId(data["source"])
+                    )
+                if "retriever" in data:
+                    new_api_key_data["retriever"] = data["retriever"]
+                api_key_collection.insert_one(new_api_key_data)
+                shared_conversations_collections.insert_one(
                     {
-                        "name": name,
-                        "key": api_uuid,
-                        "source": source,
+                        "uuid": explicit_binary,
+                        "conversation_id": {
+                            "$ref": "conversations",
+                            "$id": ObjectId(conversation_id),
+                        },
+                        "isPromptable": isPromptable,
+                        "first_n_queries": current_n_queries,
                         "user": user,
-                        "prompt_id": prompt_id,
-                        "chunks": chunks,
+                        "api_key": api_uuid,
                     }
                 )
-            shared_conversations_collections.insert_one(
-                {
-                    "uuid": explicit_binary,
-                    "conversation_id": {
-                        "$ref": "conversations",
-                        "$id": ObjectId(conversation_id),
-                    },
-                    "isPromptable": isPromptable,
-                    "first_n_queries": current_n_queries,
-                    "user": user,
-                    "api_key": api_uuid,
-                }
-            )
             ## Identifier as route parameter in frontend
             return (
                 jsonify(
@@ -672,8 +671,6 @@ def get_publicly_shared_conversations(identifier: str):
             conversation_queries = conversation["queries"][
                 : (shared["first_n_queries"])
             ]
-            for query in conversation_queries:
-                query.pop("sources")  ## avoid exposing sources
         else:
             return (
                 jsonify(
@@ -697,3 +694,466 @@ def get_publicly_shared_conversations(identifier: str):
     except Exception as err:
         print(err)
         return jsonify({"success": False, "error": str(err)}), 400
+
+
+@user.route("/api/get_message_analytics", methods=["POST"])
+def get_message_analytics():
+    data = request.get_json()
+    api_key_id = data.get("api_key_id")
+    filter_option = data.get("filter_option", "last_30_days")
+
+    try:
+        api_key = (
+            api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
+            if api_key_id
+            else None
+        )
+    except Exception as err:
+        print(err)
+        return jsonify({"success": False, "error": str(err)}), 400
+    end_date = datetime.datetime.now(datetime.timezone.utc)
+
+    if filter_option == "last_hour":
+        start_date = end_date - datetime.timedelta(hours=1)
+        group_format = "%Y-%m-%d %H:%M:00"
+        group_stage = {
+            "$group": {
+                "_id": {
+                    "minute": {
+                        "$dateToString": {"format": group_format, "date": "$date"}
+                    }
+                },
+                "total_messages": {"$sum": 1},
+            }
+        }
+
+    elif filter_option == "last_24_hour":
+        start_date = end_date - datetime.timedelta(hours=24)
+        group_format = "%Y-%m-%d %H:00"
+        group_stage = {
+            "$group": {
+                "_id": {
+                    "hour": {"$dateToString": {"format": group_format, "date": "$date"}}
+                },
+                "total_messages": {"$sum": 1},
+            }
+        }
+
+    else:
+        if filter_option in ["last_7_days", "last_15_days", "last_30_days"]:
+            filter_days = (
+                6
+                if filter_option == "last_7_days"
+                else (14 if filter_option == "last_15_days" else 29)
+            )
+        else:
+            return jsonify({"success": False, "error": "Invalid option"}), 400
+        start_date = end_date - datetime.timedelta(days=filter_days)
+        start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+        end_date = end_date.replace(hour=23, minute=59, second=59, microsecond=999999)
+        group_format = "%Y-%m-%d"
+        group_stage = {
+            "$group": {
+                "_id": {
+                    "day": {"$dateToString": {"format": group_format, "date": "$date"}}
+                },
+                "total_messages": {"$sum": 1},
+            }
+        }
+
+    try:
+        match_stage = {
+            "$match": {
+                "date": {"$gte": start_date, "$lte": end_date},
+            }
+        }
+        if api_key:
+            match_stage["$match"]["api_key"] = api_key
+        message_data = conversations_collection.aggregate(
+            [
+                match_stage,
+                group_stage,
+                {"$sort": {"_id": 1}},
+            ]
+        )
+
+        if filter_option == "last_hour":
+            intervals = generate_minute_range(start_date, end_date)
+        elif filter_option == "last_24_hour":
+            intervals = generate_hourly_range(start_date, end_date)
+        else:
+            intervals = generate_date_range(start_date, end_date)
+
+        daily_messages = {interval: 0 for interval in intervals}
+
+        for entry in message_data:
+            if filter_option == "last_hour":
+                daily_messages[entry["_id"]["minute"]] = entry["total_messages"]
+            elif filter_option == "last_24_hour":
+                daily_messages[entry["_id"]["hour"]] = entry["total_messages"]
+            else:
+                daily_messages[entry["_id"]["day"]] = entry["total_messages"]
+
+    except Exception as err:
+        print(err)
+        return jsonify({"success": False, "error": str(err)}), 400
+
+    return jsonify({"success": True, "messages": daily_messages}), 200
+
+
+@user.route("/api/get_token_analytics", methods=["POST"])
+def get_token_analytics():
+    data = request.get_json()
+    api_key_id = data.get("api_key_id")
+    filter_option = data.get("filter_option", "last_30_days")
+
+    try:
+        api_key = (
+            api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
+            if api_key_id
+            else None
+        )
+    except Exception as err:
+        print(err)
+        return jsonify({"success": False, "error": str(err)}), 400
+    end_date = datetime.datetime.now(datetime.timezone.utc)
+
+    if filter_option == "last_hour":
+        start_date = end_date - datetime.timedelta(hours=1)
+        group_format = "%Y-%m-%d %H:%M:00"
+        group_stage = {
+            "$group": {
+                "_id": {
+                    "minute": {
+                        "$dateToString": {"format": group_format, "date": "$timestamp"}
+                    }
+                },
+                "total_tokens": {
+                    "$sum": {"$add": ["$prompt_tokens", "$generated_tokens"]}
+                },
+            }
+        }
+
+    elif filter_option == "last_24_hour":
+        start_date = end_date - datetime.timedelta(hours=24)
+        group_format = "%Y-%m-%d %H:00"
+        group_stage = {
+            "$group": {
+                "_id": {
+                    "hour": {
+                        "$dateToString": {"format": group_format, "date": "$timestamp"}
+                    }
+                },
+                "total_tokens": {
+                    "$sum": {"$add": ["$prompt_tokens", "$generated_tokens"]}
+                },
+            }
+        }
+
+    else:
+        if filter_option in ["last_7_days", "last_15_days", "last_30_days"]:
+            filter_days = (
+                6
+                if filter_option == "last_7_days"
+                else (14 if filter_option == "last_15_days" else 29)
+            )
+        else:
+            return jsonify({"success": False, "error": "Invalid option"}), 400
+        start_date = end_date - datetime.timedelta(days=filter_days)
+        start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+        end_date = end_date.replace(hour=23, minute=59, second=59, microsecond=999999)
+        group_format = "%Y-%m-%d"
+        group_stage = {
+            "$group": {
+                "_id": {
+                    "day": {
+                        "$dateToString": {"format": group_format, "date": "$timestamp"}
+                    }
+                },
+                "total_tokens": {
+                    "$sum": {"$add": ["$prompt_tokens", "$generated_tokens"]}
+                },
+            }
+        }
+
+    try:
+        match_stage = {
+            "$match": {
+                "timestamp": {"$gte": start_date, "$lte": end_date},
+            }
+        }
+        if api_key:
+            match_stage["$match"]["api_key"] = api_key
+
+        token_usage_data = token_usage_collection.aggregate(
+            [
+                match_stage,
+                group_stage,
+                {"$sort": {"_id": 1}},
+            ]
+        )
+
+        if filter_option == "last_hour":
+            intervals = generate_minute_range(start_date, end_date)
+        elif filter_option == "last_24_hour":
+            intervals = generate_hourly_range(start_date, end_date)
+        else:
+            intervals = generate_date_range(start_date, end_date)
+
+        daily_token_usage = {interval: 0 for interval in intervals}
+
+        for entry in token_usage_data:
+            if filter_option == "last_hour":
+                daily_token_usage[entry["_id"]["minute"]] = entry["total_tokens"]
+            elif filter_option == "last_24_hour":
+                daily_token_usage[entry["_id"]["hour"]] = entry["total_tokens"]
+            else:
+                daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"]
+
+    except Exception as err:
+        print(err)
+        return jsonify({"success": False, "error": str(err)}), 400
+
+    return jsonify({"success": True, "token_usage": daily_token_usage}), 200
+
+
+@user.route("/api/get_feedback_analytics", methods=["POST"])
+def get_feedback_analytics():
+    data = request.get_json()
+    api_key_id = data.get("api_key_id")
+    filter_option = data.get("filter_option", "last_30_days")
+
+    try:
+        api_key = (
+            api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
+            if api_key_id
+            else None
+        )
+    except Exception as err:
+        print(err)
+        return jsonify({"success": False, "error": str(err)}), 400
+    end_date = datetime.datetime.now(datetime.timezone.utc)
+
+    if filter_option == "last_hour":
+        start_date = end_date - datetime.timedelta(hours=1)
+        group_format = "%Y-%m-%d %H:%M:00"
+        group_stage_1 = {
+            "$group": {
+                "_id": {
+                    "minute": {
+                        "$dateToString": {"format": group_format, "date": "$timestamp"}
+                    },
+                    "feedback": "$feedback",
+                },
+                "count": {"$sum": 1},
+            }
+        }
+        group_stage_2 = {
+            "$group": {
+                "_id": "$_id.minute",
+                "likes": {
+                    "$sum": {
+                        "$cond": [
+                            {"$eq": ["$_id.feedback", "LIKE"]},
+                            "$count",
+                            0,
+                        ]
+                    }
+                },
+                "dislikes": {
+                    "$sum": {
+                        "$cond": [
+                            {"$eq": ["$_id.feedback", "DISLIKE"]},
+                            "$count",
+                            0,
+                        ]
+                    }
+                },
+            }
+        }
+
+    elif filter_option == "last_24_hour":
+        start_date = end_date - datetime.timedelta(hours=24)
+        group_format = "%Y-%m-%d %H:00"
+        group_stage_1 = {
+            "$group": {
+                "_id": {
+                    "hour": {
+                        "$dateToString": {"format": group_format, "date": "$timestamp"}
+                    },
+                    "feedback": "$feedback",
+                },
+                "count": {"$sum": 1},
+            }
+        }
+        group_stage_2 = {
+            "$group": {
+                "_id": "$_id.hour",
+                "likes": {
+                    "$sum": {
+                        "$cond": [
+                            {"$eq": ["$_id.feedback", "LIKE"]},
+                            "$count",
+                            0,
+                        ]
+                    }
+                },
+                "dislikes": {
+                    "$sum": {
+                        "$cond": [
+                            {"$eq": ["$_id.feedback", "DISLIKE"]},
+                            "$count",
+                            0,
+                        ]
+                    }
+                },
+            }
+        }
+
+    else:
+        if filter_option in ["last_7_days", "last_15_days", "last_30_days"]:
+            filter_days = (
+                6
+                if filter_option == "last_7_days"
+                else (14 if filter_option == "last_15_days" else 29)
+            )
+        else:
+            return jsonify({"success": False, "error": "Invalid option"}), 400
+        start_date = end_date - datetime.timedelta(days=filter_days)
+        start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+        end_date = end_date.replace(hour=23, minute=59, second=59, microsecond=999999)
+        group_format = "%Y-%m-%d"
+        group_stage_1 = {
+            "$group": {
+                "_id": {
+                    "day": {
+                        "$dateToString": {"format": group_format, "date": "$timestamp"}
+                    },
+                    "feedback": "$feedback",
+                },
+                "count": {"$sum": 1},
+            }
+        }
+        group_stage_2 = {
+            "$group": {
+                "_id": "$_id.day",
+                "likes": {
+                    "$sum": {
+                        "$cond": [
+                            {"$eq": ["$_id.feedback", "LIKE"]},
+                            "$count",
+                            0,
+                        ]
+                    }
+                },
+                "dislikes": {
+                    "$sum": {
+                        "$cond": [
+                            {"$eq": ["$_id.feedback", "DISLIKE"]},
+                            "$count",
+                            0,
+                        ]
+                    }
+                },
+            }
+        }
+
+    try:
+        match_stage = {
+            "$match": {
+                "timestamp": {"$gte": start_date, "$lte": end_date},
+            }
+        }
+        if api_key:
+            match_stage["$match"]["api_key"] = api_key
+
+        feedback_data = feedback_collection.aggregate(
+            [
+                match_stage,
+                group_stage_1,
+                group_stage_2,
+                {"$sort": {"_id": 1}},
+            ]
+        )
+
+        if filter_option == "last_hour":
+            intervals = generate_minute_range(start_date, end_date)
+        elif filter_option == "last_24_hour":
+            intervals = generate_hourly_range(start_date, end_date)
+        else:
+            intervals = generate_date_range(start_date, end_date)
+
+        daily_feedback = {
+            interval: {"positive": 0, "negative": 0} for interval in intervals
+        }
+
+        for entry in feedback_data:
+            daily_feedback[entry["_id"]] = {
+                "positive": entry["likes"],
+                "negative": entry["dislikes"],
+            }
+
+    except Exception as err:
+        print(err)
+        return jsonify({"success": False, "error": str(err)}), 400
+
+    return jsonify({"success": True, "feedback": daily_feedback}), 200
+
+
+@user.route("/api/get_user_logs", methods=["POST"])
+def get_user_logs():
+    data = request.get_json()
+    page = int(data.get("page", 1))
+    api_key_id = data.get("api_key_id")
+    page_size = int(data.get("page_size", 10))
+    skip = (page - 1) * page_size
+
+    try:
+        api_key = (
+            api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
+            if api_key_id
+            else None
+        )
+    except Exception as err:
+        print(err)
+        return jsonify({"success": False, "error": str(err)}), 400
+
+    query = {}
+    if api_key:
+        query = {"api_key": api_key}
+    items_cursor = (
+        user_logs_collection.find(query)
+        .sort("timestamp", -1)
+        .skip(skip)
+        .limit(page_size + 1)
+    )
+    items = list(items_cursor)
+
+    results = []
+    for item in items[:page_size]:
+        results.append(
+            {
+                "id": str(item.get("_id")),
+                "action": item.get("action"),
+                "level": item.get("level"),
+                "user": item.get("user"),
+                "question": item.get("question"),
+                "sources": item.get("sources"),
+                "retriever_params": item.get("retriever_params"),
+                "timestamp": item.get("timestamp"),
+            }
+        )
+    has_more = len(items) > page_size
+
+    return (
+        jsonify(
+            {
+                "success": True,
+                "logs": results,
+                "page": page,
+                "page_size": page_size,
+                "has_more": has_more,
+            }
+        ),
+        200,
+    )

application/app.py

@@ -6,12 +6,14 @@ from application.core.settings import settings
 from application.api.user.routes import user
 from application.api.answer.routes import answer
 from application.api.internal.routes import internal
+from application.core.logging_config import setup_logging
 
 if platform.system() == "Windows":
     import pathlib
     pathlib.PosixPath = pathlib.WindowsPath
 
 dotenv.load_dotenv()
+setup_logging()
 
 app = Flask(__name__)
 app.register_blueprint(user)

application/celery_init.py

@@ -1,9 +1,15 @@
 from celery import Celery
 from application.core.settings import settings
+from celery.signals import setup_logging
 
 def make_celery(app_name=__name__):
     celery = Celery(app_name, broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND)
     celery.conf.update(settings)
     return celery
 
+@setup_logging.connect
+def config_loggers(*args, **kwargs):
+    from application.core.logging_config import setup_logging
+    setup_logging()
+
 celery = make_celery()

application/core/logging_config.py

@@ -0,0 +1,22 @@
+from logging.config import dictConfig
+
+def setup_logging():
+    dictConfig({
+        'version': 1,
+        'formatters': {
+            'default': {
+                'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+            }
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "stream": "ext://sys.stdout",
+                "formatter": "default",
+            }
+        },
+        'root': {
+            'level': 'INFO',
+            'handlers': ['console'],
+        },
+    })

application/core/settings.py

@@ -18,7 +18,7 @@ class Settings(BaseSettings):
     DEFAULT_MAX_HISTORY: int = 150
     MODEL_TOKEN_LIMITS: dict = {"gpt-3.5-turbo": 4096, "claude-2": 1e5}
     UPLOAD_FOLDER: str = "inputs"
-    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant"
+    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant" or "milvus"
     RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
 
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
@@ -29,6 +29,7 @@ class Settings(BaseSettings):
     OPENAI_API_VERSION: Optional[str] = None  # azure openai api version
     AZURE_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for answering
     AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for embeddings
+    OPENAI_BASE_URL: Optional[str] = None # openai base url for open ai compatable models
 
     # elasticsearch
     ELASTIC_CLOUD_ID: Optional[str] = None  # cloud id for elasticsearch
@@ -61,6 +62,11 @@ class Settings(BaseSettings):
     QDRANT_PATH: Optional[str] = None
     QDRANT_DISTANCE_FUNC: str = "Cosine"
 
+    # Milvus vectorstore config
+    MILVUS_COLLECTION_NAME: Optional[str] = "docsgpt"
+    MILVUS_URI: Optional[str] = "./milvus_local.db"   # milvus lite version as default
+    MILVUS_TOKEN: Optional[str] = ""
+
     BRAVE_SEARCH_API_KEY: Optional[str] = None
 
     FLASK_DEBUG_MODE: bool = False

application/llm/openai.py

@@ -2,25 +2,23 @@ from application.llm.base import BaseLLM
 from application.core.settings import settings
 
 
+
 class OpenAILLM(BaseLLM):
 
     def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
-        global openai
         from openai import OpenAI
 
         super().__init__(*args, **kwargs)
-        self.client = OpenAI(
-            api_key=api_key,
-        )
+        if settings.OPENAI_BASE_URL:
+            self.client = OpenAI(
+                api_key=api_key,
+                base_url=settings.OPENAI_BASE_URL
+            )
+        else:
+            self.client = OpenAI(api_key=api_key)
         self.api_key = api_key
         self.user_api_key = user_api_key
 
-    def _get_openai(self):
-        # Import openai when needed
-        import openai
-
-        return openai
-
     def _raw_gen(
         self,
         baseself,
@@ -29,7 +27,7 @@ class OpenAILLM(BaseLLM):
         stream=False,
         engine=settings.AZURE_DEPLOYMENT_NAME,
         **kwargs
-    ):
+    ):  
         response = self.client.chat.completions.create(
             model=model, messages=messages, stream=stream, **kwargs
         )
@@ -44,7 +42,7 @@ class OpenAILLM(BaseLLM):
         stream=True,
         engine=settings.AZURE_DEPLOYMENT_NAME,
         **kwargs
-    ):
+    ):  
         response = self.client.chat.completions.create(
             model=model, messages=messages, stream=stream, **kwargs
         )
@@ -73,8 +71,3 @@ class AzureOpenAILLM(OpenAILLM):
             api_base=settings.OPENAI_API_BASE,
             deployment_name=settings.AZURE_DEPLOYMENT_NAME,
         )
-
-    def _get_openai(self):
-        openai = super()._get_openai()
-
-        return openai

application/parser/open_ai_func.py

@@ -11,12 +11,14 @@ from retry import retry
 
 
 @retry(tries=10, delay=60)
-def store_add_texts_with_retry(store, i):
+def store_add_texts_with_retry(store, i, id):
+    # add source_id to the metadata 
+    i.metadata["source_id"] = str(id)
     store.add_texts([i.page_content], metadatas=[i.metadata])
     # store_pine.add_texts([i.page_content], metadatas=[i.metadata])
 
 
-def call_openai_api(docs, folder_name, task_status):
+def call_openai_api(docs, folder_name, id, task_status):
     # Function to create a vector store from the documents and save it to disk
 
     if not os.path.exists(f"{folder_name}"):
@@ -32,13 +34,13 @@ def call_openai_api(docs, folder_name, task_status):
         store = VectorCreator.create_vectorstore(
             settings.VECTOR_STORE,
             docs_init=docs_init,
-            path=f"{folder_name}",
+            source_id=f"{folder_name}",
             embeddings_key=os.getenv("EMBEDDINGS_KEY"),
         )
     else:
         store = VectorCreator.create_vectorstore(
             settings.VECTOR_STORE,
-            path=f"{folder_name}",
+            source_id=str(id),
             embeddings_key=os.getenv("EMBEDDINGS_KEY"),
         )
     # Uncomment for MPNet embeddings
@@ -57,7 +59,7 @@ def call_openai_api(docs, folder_name, task_status):
             task_status.update_state(
                 state="PROGRESS", meta={"current": int((c1 / s1) * 100)}
             )
-            store_add_texts_with_retry(store, i)
+            store_add_texts_with_retry(store, i, id)
         except Exception as e:
             print(e)
             print("Error on ", i)

application/parser/remote/crawler_loader.py

@@ -5,7 +5,7 @@ from application.parser.remote.base import BaseRemote
 
 class CrawlerLoader(BaseRemote):
     def __init__(self, limit=10):
-        from langchain.document_loaders import WebBaseLoader
+        from langchain_community.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader  # Initialize the document loader
         self.limit = limit  # Set the limit for the number of pages to scrape
 

application/parser/remote/sitemap_loader.py

@@ -5,7 +5,7 @@ from application.parser.remote.base import BaseRemote
 
 class SitemapLoader(BaseRemote):
     def __init__(self, limit=20):
-        from langchain.document_loaders import WebBaseLoader
+        from langchain_community.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader
         self.limit = limit  # Adding limit to control the number of URLs to process
 

application/requirements.txt

@@ -1,4 +1,4 @@
-anthropic==0.12.0
+anthropic==0.34.0
 boto3==1.34.153
 beautifulsoup4==4.12.3
 celery==5.3.6
@@ -9,26 +9,28 @@ EbookLib==0.18
 elasticsearch==8.14.0
 escodegen==1.0.11
 esprima==4.0.1
-faiss-cpu==1.7.4
-Flask==3.0.1
+Flask==3.0.3
+faiss-cpu==1.8.0.post1
 gunicorn==23.0.0
 html2text==2020.1.16
 javalang==0.13.0
-langchain==0.1.4
-langchain-openai==0.0.5
+langchain==0.2.16
+langchain-community==0.2.16
+langchain-core==0.2.38
+langchain-openai==0.1.23
 openapi3_parser==1.1.16
-pandas==2.2.0
+pandas==2.2.2
 pydantic_settings==2.4.0
-pymongo==4.6.3
+pymongo==4.8.0
 PyPDF2==3.0.1
 python-dotenv==1.0.1
-qdrant-client==1.9.0
+qdrant-client==1.11.0
 redis==5.0.1
 Requests==2.32.0
 retry==0.9.2
-sentence-transformers
-tiktoken
+sentence-transformers==3.0.1
+tiktoken==0.7.0
 torch
-tqdm==4.66.3
-transformers==4.44.0
-Werkzeug==3.0.3
+tqdm==4.66.5
+transformers==4.44.2
+Werkzeug==3.0.4

application/retriever/base.py

@@ -12,3 +12,7 @@ class BaseRetriever(ABC):
     @abstractmethod
     def search(self, *args, **kwargs):
         pass
+
+    @abstractmethod
+    def get_params(self):
+        pass

application/retriever/brave_search.py

@@ -2,7 +2,7 @@ import json
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.llm.llm_creator import LLMCreator
-from application.utils import count_tokens
+from application.utils import num_tokens_from_string
 from langchain_community.tools import BraveSearch
 
 
@@ -78,7 +78,7 @@ class BraveRetSearch(BaseRetriever):
             self.chat_history.reverse()
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
+                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
                         i["response"]
                     )
                     if tokens_current_history + tokens_batch < self.token_limit:
@@ -101,3 +101,15 @@ class BraveRetSearch(BaseRetriever):
 
     def search(self):
         return self._get_data()
+
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/classic_rag.py

@@ -1,10 +1,9 @@
-import os
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.vectorstore.vector_creator import VectorCreator
 from application.llm.llm_creator import LLMCreator
 
-from application.utils import count_tokens
+from application.utils import num_tokens_from_string
 
 
 class ClassicRAG(BaseRetriever):
@@ -21,7 +20,7 @@ class ClassicRAG(BaseRetriever):
         user_api_key=None,
     ):
         self.question = question
-        self.vectorstore = self._get_vectorstore(source=source)
+        self.vectorstore = source['active_docs'] if 'active_docs' in source else None
         self.chat_history = chat_history
         self.prompt = prompt
         self.chunks = chunks
@@ -38,21 +37,6 @@ class ClassicRAG(BaseRetriever):
         )
         self.user_api_key = user_api_key
 
-    def _get_vectorstore(self, source):
-        if "active_docs" in source:
-            if source["active_docs"].split("/")[0] == "default":
-                vectorstore = ""
-            elif source["active_docs"].split("/")[0] == "local":
-                vectorstore = "indexes/" + source["active_docs"]
-            else:
-                vectorstore = "vectors/" + source["active_docs"]
-            if source["active_docs"] == "default":
-                vectorstore = ""
-        else:
-            vectorstore = ""
-        vectorstore = os.path.join("application", vectorstore)
-        return vectorstore
-
     def _get_data(self):
         if self.chunks == 0:
             docs = []
@@ -61,13 +45,12 @@ class ClassicRAG(BaseRetriever):
                 settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
             )
             docs_temp = docsearch.search(self.question, k=self.chunks)
+            print(docs_temp)
             docs = [
                 {
-                    "title": (
-                        i.metadata["title"].split("/")[-1]
-                        if i.metadata
-                        else i.page_content
-                    ),
+                    "title": i.metadata.get(
+                        "title", i.metadata.get("post_title", i.page_content)
+                    ).split("/")[-1],
                     "text": i.page_content,
                     "source": (
                         i.metadata.get("source")
@@ -98,7 +81,7 @@ class ClassicRAG(BaseRetriever):
             self.chat_history.reverse()
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
+                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
                         i["response"]
                     )
                     if tokens_current_history + tokens_batch < self.token_limit:
@@ -121,3 +104,15 @@ class ClassicRAG(BaseRetriever):
 
     def search(self):
         return self._get_data()
+    
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.vectorstore,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/duckduck_search.py

@@ -1,7 +1,7 @@
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.llm.llm_creator import LLMCreator
-from application.utils import count_tokens
+from application.utils import num_tokens_from_string
 from langchain_community.tools import DuckDuckGoSearchResults
 from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
 
@@ -95,7 +95,7 @@ class DuckDuckSearch(BaseRetriever):
             self.chat_history.reverse()
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
+                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
                         i["response"]
                     )
                     if tokens_current_history + tokens_batch < self.token_limit:
@@ -118,3 +118,15 @@ class DuckDuckSearch(BaseRetriever):
 
     def search(self):
         return self._get_data()
+    
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/retriever_creator.py

@@ -5,15 +5,16 @@ from application.retriever.brave_search import BraveRetSearch
 
 
 class RetrieverCreator:
-    retievers = {
+    retrievers = {
         'classic': ClassicRAG,
         'duckduck_search': DuckDuckSearch,
-        'brave_search': BraveRetSearch
+        'brave_search': BraveRetSearch,
+        'default': ClassicRAG
     }
 
     @classmethod
     def create_retriever(cls, type, *args, **kwargs):
-        retiever_class = cls.retievers.get(type.lower())
+        retiever_class = cls.retrievers.get(type.lower())
         if not retiever_class:
             raise ValueError(f"No retievers class found for type {type}")
         return retiever_class(*args, **kwargs)

application/usage.py

@@ -2,7 +2,7 @@ import sys
 from pymongo import MongoClient
 from datetime import datetime
 from application.core.settings import settings
-from application.utils import count_tokens
+from application.utils import num_tokens_from_string
 
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
@@ -24,9 +24,9 @@ def update_token_usage(user_api_key, token_usage):
 def gen_token_usage(func):
     def wrapper(self, model, messages, stream, **kwargs):
         for message in messages:
-            self.token_usage["prompt_tokens"] += count_tokens(message["content"])
+            self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
         result = func(self, model, messages, stream, **kwargs)
-        self.token_usage["generated_tokens"] += count_tokens(result)
+        self.token_usage["generated_tokens"] += num_tokens_from_string(result)
         update_token_usage(self.user_api_key, self.token_usage)
         return result
 
@@ -36,14 +36,14 @@ def gen_token_usage(func):
 def stream_token_usage(func):
     def wrapper(self, model, messages, stream, **kwargs):
         for message in messages:
-            self.token_usage["prompt_tokens"] += count_tokens(message["content"])
+            self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
         batch = []
         result = func(self, model, messages, stream, **kwargs)
         for r in result:
             batch.append(r)
             yield r
         for line in batch:
-            self.token_usage["generated_tokens"] += count_tokens(line)
+            self.token_usage["generated_tokens"] += num_tokens_from_string(line)
         update_token_usage(self.user_api_key, self.token_usage)
 
     return wrapper

application/utils.py

@@ -1,6 +1,22 @@
-from transformers import GPT2TokenizerFast
+import tiktoken
 
-tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
-tokenizer.model_max_length = 100000
-def count_tokens(string):
-    return len(tokenizer(string)['input_ids'])
+_encoding = None
+
+def get_encoding():
+    global _encoding
+    if _encoding is None:
+        _encoding = tiktoken.get_encoding("cl100k_base")
+    return _encoding
+
+def num_tokens_from_string(string: str) -> int:
+    encoding = get_encoding()
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
+
+def count_tokens_docs(docs):
+    docs_content = ""
+    for doc in docs:
+        docs_content += doc.page_content
+
+    tokens = num_tokens_from_string(docs_content)
+    return tokens

application/vectorstore/base.py

@@ -1,13 +1,30 @@
 from abc import ABC, abstractmethod
 import os
-from langchain_community.embeddings import (
-    HuggingFaceEmbeddings,
-    CohereEmbeddings,
-    HuggingFaceInstructEmbeddings,
-)
+from sentence_transformers import SentenceTransformer
 from langchain_openai import OpenAIEmbeddings
 from application.core.settings import settings
 
+class EmbeddingsWrapper:
+    def __init__(self, model_name, *args, **kwargs):
+        self.model = SentenceTransformer(model_name, config_kwargs={'allow_dangerous_deserialization': True}, *args, **kwargs)
+        self.dimension = self.model.get_sentence_embedding_dimension()
+
+    def embed_query(self, query: str):
+        return self.model.encode(query).tolist()
+    
+    def embed_documents(self, documents: list):
+        return self.model.encode(documents).tolist()
+    
+    def __call__(self, text):
+        if isinstance(text, str):
+            return self.embed_query(text)
+        elif isinstance(text, list):
+            return self.embed_documents(text)
+        else:
+            raise ValueError("Input must be a string or a list of strings")
+
+
+
 class EmbeddingsSingleton:
     _instances = {}
 
@@ -23,16 +40,15 @@ class EmbeddingsSingleton:
     def _create_instance(embeddings_name, *args, **kwargs):
         embeddings_factory = {
             "openai_text-embedding-ada-002": OpenAIEmbeddings,
-            "huggingface_sentence-transformers/all-mpnet-base-v2": HuggingFaceEmbeddings,
-            "huggingface_sentence-transformers-all-mpnet-base-v2": HuggingFaceEmbeddings,
-            "huggingface_hkunlp/instructor-large": HuggingFaceInstructEmbeddings,
-            "cohere_medium": CohereEmbeddings
+            "huggingface_sentence-transformers/all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
+            "huggingface_sentence-transformers-all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
+            "huggingface_hkunlp/instructor-large": lambda: EmbeddingsWrapper("hkunlp/instructor-large"),
         }
 
-        if embeddings_name not in embeddings_factory:
-            raise ValueError(f"Invalid embeddings_name: {embeddings_name}")
-
-        return embeddings_factory[embeddings_name](*args, **kwargs)
+        if embeddings_name in embeddings_factory:
+            return embeddings_factory[embeddings_name](*args, **kwargs)
+        else:
+            return EmbeddingsWrapper(embeddings_name, *args, **kwargs)
 
 class BaseVectorStore(ABC):
     def __init__(self):
@@ -58,22 +74,14 @@ class BaseVectorStore(ABC):
                     embeddings_name,
                     openai_api_key=embeddings_key
                 )
-        elif embeddings_name == "cohere_medium":
-            embedding_instance = EmbeddingsSingleton.get_instance(
-                embeddings_name,
-                cohere_api_key=embeddings_key
-            )
         elif embeddings_name == "huggingface_sentence-transformers/all-mpnet-base-v2":
             if os.path.exists("./model/all-mpnet-base-v2"):
                 embedding_instance = EmbeddingsSingleton.get_instance(
-                    embeddings_name,
-                    model_name="./model/all-mpnet-base-v2",
-                    model_kwargs={"device": "cpu"}
+                    embeddings_name="./model/all-mpnet-base-v2",
                 )
             else:
                 embedding_instance = EmbeddingsSingleton.get_instance(
                     embeddings_name,
-                    model_kwargs={"device": "cpu"}
                 )
         else:
             embedding_instance = EmbeddingsSingleton.get_instance(embeddings_name)

application/vectorstore/elasticsearch.py

@@ -9,9 +9,9 @@ import elasticsearch
 class ElasticsearchStore(BaseVectorStore):
     _es_connection = None  # Class attribute to hold the Elasticsearch connection
 
-    def __init__(self, path, embeddings_key, index_name=settings.ELASTIC_INDEX):
+    def __init__(self, source_id, embeddings_key, index_name=settings.ELASTIC_INDEX):
         super().__init__()
-        self.path = path.replace("application/indexes/", "").rstrip("/")
+        self.source_id = source_id.replace("application/indexes/", "").rstrip("/")
         self.embeddings_key = embeddings_key
         self.index_name = index_name
         
@@ -81,7 +81,7 @@ class ElasticsearchStore(BaseVectorStore):
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
         vector = embeddings.embed_query(question)
         knn = {
-            "filter": [{"match": {"metadata.store.keyword": self.path}}],
+            "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
             "field": "vector",
             "k": k,
             "num_candidates": 100,
@@ -100,7 +100,7 @@ class ElasticsearchStore(BaseVectorStore):
                             }
                         }
                     ],
-                    "filter": [{"match": {"metadata.store.keyword": self.path}}],
+                    "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
                 }
             },
             "rank": {"rrf": {}},
@@ -209,5 +209,4 @@ class ElasticsearchStore(BaseVectorStore):
 
     def delete_index(self):
         self._es_connection.delete_by_query(index=self.index_name, query={"match": {
-                                      "metadata.store.keyword": self.path}},)
-
+                                      "metadata.source_id.keyword": self.source_id}},)

application/vectorstore/faiss.py

@@ -1,12 +1,22 @@
 from langchain_community.vectorstores import FAISS
 from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
+import os
+
+def get_vectorstore(path):
+    if path:
+        vectorstore = "indexes/"+path
+        vectorstore = os.path.join("application", vectorstore)
+    else:
+        vectorstore = os.path.join("application")
+
+    return vectorstore
 
 class FaissStore(BaseVectorStore):
 
-    def __init__(self, path, embeddings_key, docs_init=None):
+    def __init__(self, source_id, embeddings_key, docs_init=None):
         super().__init__()
-        self.path = path
+        self.path = get_vectorstore(source_id)
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
         if docs_init:
             self.docsearch = FAISS.from_documents(
@@ -14,7 +24,8 @@ class FaissStore(BaseVectorStore):
             )
         else:
             self.docsearch = FAISS.load_local(
-                self.path, embeddings
+                self.path, embeddings, 
+                allow_dangerous_deserialization=True
             )
         self.assert_embedding_dimensions(embeddings)
 
@@ -37,10 +48,10 @@ class FaissStore(BaseVectorStore):
         """
         if settings.EMBEDDINGS_NAME == "huggingface_sentence-transformers/all-mpnet-base-v2":
             try:
-                word_embedding_dimension = embeddings.client[1].word_embedding_dimension
+                word_embedding_dimension = embeddings.dimension
             except AttributeError as e:
-                raise AttributeError("word_embedding_dimension not found in embeddings.client[1]") from e
+                raise AttributeError("'dimension' attribute not found in embeddings instance. Make sure the embeddings object is properly initialized.") from e
             docsearch_index_dimension = self.docsearch.index.d
             if word_embedding_dimension != docsearch_index_dimension:
-                raise ValueError(f"word_embedding_dimension ({word_embedding_dimension}) " +
-                                 f"!= docsearch_index_word_embedding_dimension ({docsearch_index_dimension})")
+                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) " +
+                                 f"!= docsearch index dimension ({docsearch_index_dimension})")

application/vectorstore/milvus.py

@@ -0,0 +1,37 @@
+from typing import List, Optional
+from uuid import uuid4
+
+
+from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
+
+
+class MilvusStore(BaseVectorStore):
+    def __init__(self, path: str = "", embeddings_key: str = "embeddings"):
+        super().__init__()
+        from langchain_milvus import Milvus
+
+        connection_args = {
+            "uri": settings.MILVUS_URI,
+            "token": settings.MILVUS_TOKEN,
+        }
+        self._docsearch = Milvus(
+            embedding_function=self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key),
+            collection_name=settings.MILVUS_COLLECTION_NAME,
+            connection_args=connection_args,
+        )
+        self._path = path
+
+    def search(self, question, k=2, *args, **kwargs):
+        return self._docsearch.similarity_search(query=question, k=k, filter={"path": self._path} *args, **kwargs)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]], *args, **kwargs):
+        ids = [str(uuid4()) for _ in range(len(texts))]
+
+        return self._docsearch.add_texts(texts=texts, metadatas=metadatas, ids=ids, *args, **kwargs)
+
+    def save_local(self, *args, **kwargs):
+        pass
+
+    def delete_index(self, *args, **kwargs):
+        pass

application/vectorstore/mongodb.py

@@ -5,7 +5,7 @@ from application.vectorstore.document_class import Document
 class MongoDBVectorStore(BaseVectorStore):
     def __init__(
         self,
-        path: str = "",
+        source_id: str = "",
         embeddings_key: str = "embeddings",
         collection: str = "documents",
         index_name: str = "vector_search_index",
@@ -18,7 +18,7 @@ class MongoDBVectorStore(BaseVectorStore):
         self._embedding_key = embedding_key
         self._embeddings_key = embeddings_key
         self._mongo_uri = settings.MONGO_URI
-        self._path = path.replace("application/indexes/", "").rstrip("/")
+        self._source_id = source_id.replace("application/indexes/", "").rstrip("/")
         self._embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
 
         try:
@@ -46,7 +46,7 @@ class MongoDBVectorStore(BaseVectorStore):
                     "numCandidates": k * 10, 
                     "index": self._index_name,
                     "filter": {
-                        "store": {"$eq": self._path}
+                        "source_id": {"$eq": self._source_id}
                     }
                 }
             }
@@ -123,4 +123,4 @@ class MongoDBVectorStore(BaseVectorStore):
         return result_ids
     
     def delete_index(self, *args, **kwargs):
-        self._collection.delete_many({"store": self._path})
+        self._collection.delete_many({"source_id": self._source_id})

application/vectorstore/qdrant.py

@@ -5,12 +5,12 @@ from qdrant_client import models
 
 
 class QdrantStore(BaseVectorStore):
-    def __init__(self, path: str = "", embeddings_key: str = "embeddings"):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
         self._filter = models.Filter(
             must=[
                 models.FieldCondition(
-                    key="metadata.store",
-                    match=models.MatchValue(value=path.replace("application/indexes/", "").rstrip("/")),
+                    key="metadata.source_id",
+                    match=models.MatchValue(value=source_id.replace("application/indexes/", "").rstrip("/")),
                 )
             ]
         )

application/vectorstore/vector_creator.py

@@ -1,5 +1,6 @@
 from application.vectorstore.faiss import FaissStore
 from application.vectorstore.elasticsearch import ElasticsearchStore
+from application.vectorstore.milvus import MilvusStore
 from application.vectorstore.mongodb import MongoDBVectorStore
 from application.vectorstore.qdrant import QdrantStore
 
@@ -10,6 +11,7 @@ class VectorCreator:
         "elasticsearch": ElasticsearchStore,
         "mongodb": MongoDBVectorStore,
         "qdrant": QdrantStore,
+        "milvus": MilvusStore,
     }
 
     @classmethod

application/worker.py

@@ -2,10 +2,11 @@ import os
 import shutil
 import string
 import zipfile
-import tiktoken
 from urllib.parse import urljoin
+import logging
 
 import requests
+from bson.objectid import ObjectId
 
 from application.core.settings import settings
 from application.parser.file.bulk import SimpleDirectoryReader
@@ -13,11 +14,13 @@ from application.parser.remote.remote_creator import RemoteCreator
 from application.parser.open_ai_func import call_openai_api
 from application.parser.schema.base import Document
 from application.parser.token_func import group_split
+from application.utils import count_tokens_docs
+
+
 
 # Define a function to extract metadata from a given filename.
 def metadata_from_filename(title):
-    store = "/".join(title.split("/")[1:3])
-    return {"title": title, "store": store}
+    return {"title": title}
 
 
 # Define a function to generate a random string of a given length.
@@ -25,9 +28,7 @@ def generate_random_string(length):
     return "".join([string.ascii_letters[i % 52] for i in range(length)])
 
 
-current_dir = os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-)
+current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
 def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
@@ -41,7 +42,7 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
         max_depth (int): Maximum allowed depth of recursion to prevent infinite loops.
     """
     if current_depth > max_depth:
-        print(f"Reached maximum recursion depth of {max_depth}")
+        logging.warning(f"Reached maximum recursion depth of {max_depth}")
         return
 
     with zipfile.ZipFile(zip_path, "r") as zip_ref:
@@ -58,7 +59,7 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
 
 
 # Define the main function for ingesting and processing documents.
-def ingest_worker(self, directory, formats, name_job, filename, user):
+def ingest_worker(self, directory, formats, name_job, filename, user, retriever="classic"):
     """
     Ingest and process documents.
 
@@ -69,6 +70,7 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         name_job (str): Name of the job for this ingestion task.
         filename (str): Name of the file to be ingested.
         user (str): Identifier for the user initiating the ingestion.
+        retriever (str): Type of retriever to use for processing the documents.
 
     Returns:
         dict: Information about the completed ingestion task, including input parameters and a "limited" flag.
@@ -88,16 +90,13 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
     max_tokens = 1250
     recursion_depth = 2
     full_path = os.path.join(directory, user, name_job)
-    import sys
 
-    print(full_path, file=sys.stderr)
+    logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job})
     # check if API_URL env variable is set
     file_data = {"name": name_job, "file": filename, "user": user}
     response = requests.get(
         urljoin(settings.API_URL, "/api/download"), params=file_data
     )
-    # check if file is in the response
-    print(response, file=sys.stderr)
     file = response.content
 
     if not os.path.exists(full_path):
@@ -107,9 +106,7 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
 
     # check if file is .zip and extract it
     if filename.endswith(".zip"):
-        extract_zip_recursive(
-            os.path.join(full_path, filename), full_path, 0, recursion_depth
-        )
+        extract_zip_recursive(os.path.join(full_path, filename), full_path, 0, recursion_depth)
 
     self.update_state(state="PROGRESS", meta={"current": 1})
 
@@ -130,33 +127,27 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
     )
 
     docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+    id = ObjectId()
 
-    call_openai_api(docs, full_path, self)
+    call_openai_api(docs, full_path, id, self)
     tokens = count_tokens_docs(docs)
     self.update_state(state="PROGRESS", meta={"current": 100})
 
     if sample:
         for i in range(min(5, len(raw_docs))):
-            print(raw_docs[i].text)
+            logging.info(f"Sample document {i}: {raw_docs[i]}")
 
     # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
     # and send them to the server (provide user and name in form)
-    file_data = {"name": name_job, "user": user, "tokens":tokens}
+    file_data = {"name": name_job, "user": user, "tokens": tokens, "retriever": retriever, "id": str(id), 'type': 'local'}
     if settings.VECTOR_STORE == "faiss":
         files = {
             "file_faiss": open(full_path + "/index.faiss", "rb"),
             "file_pkl": open(full_path + "/index.pkl", "rb"),
         }
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
-        )
-        response = requests.get(
-            urljoin(settings.API_URL, "/api/delete_old?path=" + full_path)
-        )
+        response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data)
     else:
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), data=file_data
-        )
+        response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
 
     # delete local
     shutil.rmtree(full_path)
@@ -171,7 +162,7 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
     }
 
 
-def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
+def remote_worker(self, source_data, name_job, user, loader, directory="temp", retriever="classic"):
     token_check = True
     min_tokens = 150
     max_tokens = 1250
@@ -180,6 +171,7 @@ def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
     if not os.path.exists(full_path):
         os.makedirs(full_path)
     self.update_state(state="PROGRESS", meta={"current": 1})
+    logging.info(f"Remote job: {full_path}", extra={"user": user, "job": name_job, source_data: source_data})
 
     remote_loader = RemoteCreator.create_loader(loader)
     raw_docs = remote_loader.load_data(source_data)
@@ -191,47 +183,24 @@ def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
         token_check=token_check,
     )
     # docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
-    call_openai_api(docs, full_path, self)
     tokens = count_tokens_docs(docs)
+    id = ObjectId()
+    call_openai_api(docs, full_path, id, self)
     self.update_state(state="PROGRESS", meta={"current": 100})
 
     # Proceed with uploading and cleaning as in the original function
-    file_data = {"name": name_job, "user": user, "tokens":tokens}
+    file_data = {"name": name_job, "user": user, "tokens": tokens, "retriever": retriever, 
+                 "id": str(id), 'type': loader, 'remote_data': source_data}
     if settings.VECTOR_STORE == "faiss":
         files = {
             "file_faiss": open(full_path + "/index.faiss", "rb"),
             "file_pkl": open(full_path + "/index.pkl", "rb"),
         }
-        
-        requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
-        )
-        requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path))
+
+        requests.post(urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data)
     else:
         requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
 
     shutil.rmtree(full_path)
 
     return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}
-
-
-def count_tokens_docs(docs):
-    # Here we convert the docs list to a string and calculate the number of tokens the string represents.
-    # docs_content = (" ".join(docs))
-    docs_content = ""
-    for doc in docs:
-        docs_content += doc.page_content
-
-    tokens, total_price = num_tokens_from_string(
-        string=docs_content, encoding_name="cl100k_base"
-    )
-    # Here we print the number of tokens and the approx user cost with some visually appealing formatting.
-    return tokens
-
-
-def num_tokens_from_string(string: str, encoding_name: str) -> int:
-    # Function to convert string to tokens and estimate user cost.
-    encoding = tiktoken.get_encoding(encoding_name)
-    num_tokens = len(encoding.encode(string))
-    total_price = (num_tokens / 1000) * 0.0004
-    return num_tokens, total_price

docker-compose-azure.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docker-compose-dev.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
 
   redis:

docker-compose-local.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docker-compose-mock.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docker-compose.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docs/package.json

@@ -7,8 +7,8 @@
   "license": "MIT",
   "dependencies": {
     "@vercel/analytics": "^1.1.1",
-    "docsgpt": "^0.3.7",
-    "next": "^14.1.1",
+    "docsgpt": "^0.4.1",
+    "next": "^14.2.12",
     "nextra": "^2.13.2",
     "nextra-theme-docs": "^2.13.2",
     "react": "^18.2.0",

docs/pages/Deploying/Quickstart.md

@@ -67,46 +67,3 @@ To run the setup on Windows, you have two options: using the Windows Subsystem f
 
 These steps should help you set up and run the project on Windows using either WSL or Git Bash/Command Prompt. 
 **Important:** Ensure that Docker is installed and properly configured on your Windows system for these steps to work.
-
-
-For WINDOWS:
-
-To run the given setup on Windows, you can use the Windows Subsystem for Linux (WSL) or a Git Bash terminal to execute similar commands. Here are the steps adapted for Windows:
-
-Option 1: Using Windows Subsystem for Linux (WSL):
-
-1. Install WSL if you haven't already. You can follow the official Microsoft documentation for installation: (https://learn.microsoft.com/en-us/windows/wsl/install).
-2. After setting up WSL, open the WSL terminal.
-3. Clone the repository and create the `.env` file:
-   ```bash
-   git clone https://github.com/arc53/DocsGPT.git
-   cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
-   ```
-4. Run the following command to start the setup with Docker Compose:
-    ```bash
-    ./run-with-docker-compose.sh
-    ```
-5. Open your web browser and navigate to http://localhost:5173/.
-6. To stop the setup, just press **Ctrl + C** in the WSL terminal.
-
-Option 2: Using Git Bash or Command Prompt (CMD):
-
-1. Install Git for Windows if you haven't already. You can download it from the official website: (https://gitforwindows.org/).
-2. Open Git Bash or Command Prompt.
-3. Clone the repository and create the `.env` file:
-   ```bash
-   git clone https://github.com/arc53/DocsGPT.git
-   cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
-   ```
-4. Run the following command to start the setup with Docker Compose:
-   ```bash
-   ./run-with-docker-compose.sh
-   ```
-5. Open your web browser and navigate to http://localhost:5173/.
-6. To stop the setup, just press **Ctrl + C** in the Git Bash or Command Prompt terminal.
-
-These steps should help you set up and run the project on Windows using either WSL or Git Bash/Command Prompt. Make sure you have Docker installed and properly configured on your Windows system for this to work.

docs/pages/Extensions/react-widget.md

@@ -17,25 +17,31 @@ Now, you can use the widget in your component like this :
 ```jsx
 <DocsGPTWidget
   apiHost="https://your-docsgpt-api.com"
-  selectDocs="local/docs.zip"
   apiKey=""
-  avatar = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png",
-  title = "Get AI assistance",
-  description = "DocsGPT's AI Chatbot is here to help",
-  heroTitle = "Welcome to DocsGPT !",
+  avatar = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png"
+  title = "Get AI assistance"
+  description = "DocsGPT's AI Chatbot is here to help"
+  heroTitle = "Welcome to DocsGPT !"
   heroDescription="This chatbot is built with DocsGPT and utilises GenAI, 
   please review important information using sources."
+  theme = "dark"
+  buttonIcon = "https://your-icon"
+  buttonBg = "#222327"
 />
 ```
-DocsGPTWidget takes 8 **props** with default fallback values:
+To tailor the widget to your needs, you can configure the following props in your component:
 1. `apiHost` — The URL of your DocsGPT API.
-2. `selectDocs` — The documentation source that you want to use for your widget (e.g. `default` or `local/docs1.zip`).
+2. `theme` — Allows to select your specific theme (dark or light).
 3. `apiKey` — Usually, it's empty.
 4. `avatar`: Specifies the URL of the avatar or image representing the chatbot.
 5. `title`: Sets the title text displayed in the chatbot interface.
 6. `description`: Provides a brief description of the chatbot's purpose or functionality.
 7. `heroTitle`: Displays a welcome title when users interact with the chatbot.
 8. `heroDescription`: Provide additional introductory text or information about the chatbot's capabilities.
+9. `buttonIcon`: Specifies the url of the icon image for the widget.
+10. `buttonBg`: Allows to specify the Background color of the widget.
+11. `size`: Sets the size of the widget ( small, medium).
+
 
 ### How to use DocsGPTWidget with [Nextra](https://nextra.site/) (Next.js + MDX)
 Install your widget as described above and then go to your `pages/` folder and create a new file `_app.js` with the following content:
@@ -55,22 +61,30 @@ export default function MyApp({ Component, pageProps }) {
 ```html
 <!DOCTYPE html>
 <html lang="en">
-<head>
-  <meta charset="UTF-8" />
-  <meta http-equiv="X-UA-Compatible" content="IE=edge" />
-  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>DocsGPT Widget</title>
-</head>
-<body>
-  <div id="app"></div>
-  <!-- Include the widget script from dist/modern or dist/legacy -->
-  <script src="https://unpkg.com/docsgpt/dist/modern/main.js" type="module"></script>
-  <script type="module">
-    window.onload = function() {
-      renderDocsGPTWidget('app');
-    }
-  </script>
-</body>
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta http-equiv="X-UA-Compatible" content="ie=edge" />
+    <title>HTML + CSS</title>
+    <link rel="stylesheet" href="styles.css" />
+  </head>
+  <body>
+    <h1>This is a simple HTML + CSS template!</h1>
+    <div id="app"></div>
+    <!-- Include the widget script from dist/modern or dist/legacy -->
+    <script
+      src="https://unpkg.com/docsgpt/dist/modern/main.js"
+      type="module"
+    ></script>
+    <script type="module">
+      window.onload = function () {
+        renderDocsGPTWidget("app", {
+          apiKey: "",
+          size: "medium",
+        });
+      };
+    </script>
+  </body>
 </html>
 ```
 To link the widget to your api and your documents you can pass parameters to the renderDocsGPTWidget('div id', { parameters }).
@@ -82,22 +96,24 @@ To link the widget to your api and your documents you can pass parameters to the
   <meta http-equiv="X-UA-Compatible" content="IE=edge" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <title>DocsGPT Widget</title>
+  <script src="https://unpkg.com/docsgpt/dist/modern/main.js" type="module"></script>  
 </head>
 <body>
   <div id="app"></div>
   <!-- Include the widget script from dist/modern or dist/legacy -->
-  <script src="https://unpkg.com/docsgpt/dist/modern/main.js" type="module"></script>
   <script type="module">
     window.onload = function() {
       renderDocsGPTWidget('app', {
         apiHost: 'http://localhost:7001',
-        selectDocs: 'default',
-        apiKey: '',
+        apiKey:"",
         avatar: 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
         title: 'Get AI assistance',
         description: "DocsGPT's AI Chatbot is here to help",
         heroTitle: 'Welcome to DocsGPT!',
-        heroDescription: 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.'
+        heroDescription: 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.',
+        theme:"dark",
+        buttonIcon:"https://your-icon",
+        buttonBg:"#222327"
       });
     }
   </script>

docs/pages/Guides/How-to-use-different-LLM.mdx

@@ -36,6 +36,14 @@ List of latest supported LLMs are https://github.com/arc53/DocsGPT/blob/main/app
 Visit application/llm and select the file of your selected llm and there you will find the speicifc requirements needed to be filled in order to use it,i.e API key of that llm.
 </Steps>
 
+### For OpenAI-Compatible Endpoints:
+DocsGPT supports the use of OpenAI-compatible endpoints through base URL substitution. This feature allows you to use alternative AI models or services that implement the OpenAI API interface.
+
+
+Set the OPENAI_BASE_URL in your environment. You can change .env file with OPENAI_BASE_URL with the desired base URL or docker-compose.yml file and add the environment variable to the backend container.
+
+> Make sure you have the right API_KEY and correct LLM_NAME.
+
 
 
 

docs/pages/_app.mdx

@@ -4,7 +4,7 @@ export default function MyApp({ Component, pageProps }) {
   return (
     <>
       <Component {...pageProps} />
-        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" />
+        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" theme="dark" />
     </>
   )
 }

extensions/chrome/package-lock.json

@@ -107,12 +107,12 @@
       }
     },
     "node_modules/braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dev": true,
       "dependencies": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       },
       "engines": {
         "node": ">=8"
@@ -260,9 +260,9 @@
       }
     },
     "node_modules/fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "dev": true,
       "dependencies": {
         "to-regex-range": "^5.0.1"
@@ -884,12 +884,12 @@
       "dev": true
     },
     "braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dev": true,
       "requires": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       }
     },
     "camelcase-css": {
@@ -1000,9 +1000,9 @@
       }
     },
     "fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "dev": true,
       "requires": {
         "to-regex-range": "^5.0.1"

extensions/react-widget/README.md

@@ -27,15 +27,18 @@ To link the widget to your api and your documents you can pass parameters to the
 
     const App = () => {
       return <DocsGPTWidget
-             apiHost = 'http://localhost:7001',
-             selectDocs = 'default',
-             apiKey = '',
-             avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
-             title = 'Get AI assistance',
-             description = 'DocsGPT\'s AI Chatbot is here to help',
-             heroTitle = 'Welcome to DocsGPT !',
-             heroDescription='This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.'
-             />;
+               apiHost="https://your-docsgpt-api.com"
+               apiKey=""
+               avatar = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png"
+               title = "Get AI assistance"
+               description = "DocsGPT's AI Chatbot is here to help"
+               heroTitle = "Welcome to DocsGPT !"
+               heroDescription="This chatbot is built with DocsGPT and utilises GenAI, 
+               please review important information using sources."
+               theme = "dark"
+               buttonIcon = "https://your-icon"
+               buttonBg = "#222327"
+          />;
     };
 ```
 
@@ -80,15 +83,17 @@ To link the widget to your api and your documents you can pass parameters to the
         <script src="https://unpkg.com/docsgpt/dist/modern/main.js" type="module"></script>
         <script type="module">
           window.onload = function() {
-            renderDocsGPTWidget('app', , {
+            renderDocsGPTWidget('app', {
               apiHost: 'http://localhost:7001',
-              selectDocs: 'default',
-              apiKey: '',
+              apiKey:"",
               avatar: 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
               title: 'Get AI assistance',
               description: "DocsGPT's AI Chatbot is here to help",
-              heroTitle: 'Welcome to DocsGPT !',
-              heroDescription: 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.'
+              heroTitle: 'Welcome to DocsGPT!',
+              heroDescription: 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.',
+              theme:"dark",
+              buttonIcon:"https://your-icon.svg",
+              buttonBg:"#222327"
             });
           }
         </script>

extensions/react-widget/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docsgpt",
-  "version": "0.3.9",
+  "version": "0.4.2",
   "private": false,
   "description": "DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.",
   "source": "./src/index.html",
@@ -31,6 +31,7 @@
   },
   "scripts": {
     "build": "parcel build src/main.tsx --public-url ./",
+    "build:react": "parcel build src/index.ts",
     "dev": "parcel src/index.html -p 3000",
     "test": "jest",
     "lint": "eslint",
@@ -39,7 +40,6 @@
   },
   "dependencies": {
     "@babel/plugin-transform-flow-strip-types": "^7.23.3",
-    "@bpmn-io/snarkdown": "^2.2.0",
     "@parcel/resolver-glob": "^2.12.0",
     "@parcel/transformer-svg-react": "^2.12.0",
     "@parcel/transformer-typescript-tsc": "^2.12.0",
@@ -51,6 +51,7 @@
     "flow-bin": "^0.229.2",
     "i": "^0.3.7",
     "install": "^0.13.0",
+    "markdown-it": "^14.1.0",
     "npm": "^10.5.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
@@ -63,6 +64,7 @@
     "@parcel/packager-ts": "^2.12.0",
     "@parcel/transformer-typescript-types": "^2.12.0",
     "@types/dompurify": "^3.0.5",
+    "@types/markdown-it": "^14.1.2",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "babel-loader": "^8.0.4",

extensions/react-widget/publish.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+## chmod +x publish.sh - to upgrade ownership
+set -e
+cat package.json >> package_copy.json
+cat package-lock.json >> package-lock_copy.json
+publish_package() {
+  PACKAGE_NAME=$1
+  BUILD_COMMAND=$2
+  # Update package name in package.json
+  jq --arg name "$PACKAGE_NAME" '.name=$name' package.json > temp.json && mv temp.json package.json
+
+  # Remove 'target' key if the package name is 'docsgpt-react'
+  if [ "$PACKAGE_NAME" = "docsgpt-react" ]; then
+    jq 'del(.targets)' package.json > temp.json && mv temp.json package.json
+  fi
+
+  if [ -d "dist" ]; then
+    echo "Deleting existing dist directory..."
+    rm -rf dist
+  fi
+
+  npm version patch
+
+  npm run "$BUILD_COMMAND"
+
+  # Publish to npm
+  npm publish
+  # Clean up
+  mv package_copy.json package.json
+  mv package-lock_copy.json package-lock.json
+  echo "Published ${PACKAGE_NAME}"
+}
+
+# Publish docsgpt package
+publish_package "docsgpt" "build"
+
+# Publish docsgpt-react package
+publish_package "docsgpt-react" "build:react"
+
+
+rm -rf package_copy.json
+rm -rf package-lock_copy.json
+echo "---Process completed---"

extensions/react-widget/src/assets/dislike.svg

@@ -0,0 +1,4 @@
+<svg width="14" height="14" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.37776 10.1001V12.9C6.37776 13.457 6.599 13.9911 6.99282 14.3849C7.38664 14.7788 7.92077 15 8.47772 15L11.2777 8.70011V1.00025H3.38181C3.04419 0.996436 2.71656 1.11477 2.45929 1.33344C2.20203 1.55212 2.03246 1.8564 1.98184 2.19023L1.01585 8.49012C0.985398 8.69076 0.998931 8.89563 1.05551 9.09053C1.1121 9.28543 1.21038 9.46569 1.34355 9.61884C1.47671 9.77198 1.64159 9.89434 1.82674 9.97744C2.01189 10.0605 2.2129 10.1024 2.41583 10.1001H6.37776ZM11.2777 1.00025H13.1466C13.5428 0.993247 13.9277 1.13195 14.2284 1.39002C14.5291 1.64809 14.7245 2.00758 14.7776 2.40023V7.30014C14.7245 7.69279 14.5291 8.05227 14.2284 8.31035C13.9277 8.56842 13.5428 8.70712 13.1466 8.70011H11.2777" fill="none"/>
+<path d="M11.2777 8.70011L8.47772 15C7.92077 15 7.38664 14.7788 6.99282 14.3849C6.599 13.9911 6.37776 13.457 6.37776 12.9V10.1001H2.41583C2.2129 10.1024 2.01189 10.0605 1.82674 9.97744C1.64159 9.89434 1.47671 9.77198 1.34355 9.61884C1.21038 9.46569 1.1121 9.28543 1.05551 9.09053C0.998931 8.89563 0.985398 8.69076 1.01585 8.49012L1.98184 2.19023C2.03246 1.8564 2.20203 1.55212 2.45929 1.33344C2.71656 1.11477 3.04419 0.996436 3.38181 1.00025H11.2777M11.2777 8.70011V1.00025M11.2777 8.70011H13.1466C13.5428 8.70712 13.9277 8.56842 14.2284 8.31035C14.5291 8.05227 14.7245 7.69279 14.7776 7.30014V2.40023C14.7245 2.00758 14.5291 1.64809 14.2284 1.39002C13.9277 1.13195 13.5428 0.993247 13.1466 1.00025H11.2777" stroke="current" stroke-width="1.4" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

extensions/react-widget/src/assets/like.svg

@@ -0,0 +1,4 @@
+<svg width="14" height="14" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
+<path d="M9.39995 5.89997V3.09999C9.39995 2.54304 9.1787 2.0089 8.78487 1.61507C8.39105 1.22125 7.85691 1 7.29996 1L4.49998 7.29996V14.9999H12.3959C12.7336 15.0037 13.0612 14.8854 13.3185 14.6667C13.5757 14.448 13.7453 14.1437 13.7959 13.8099L14.7619 7.50996C14.7924 7.30931 14.7788 7.10444 14.7222 6.90954C14.6657 6.71464 14.5674 6.53437 14.4342 6.38123C14.301 6.22808 14.1362 6.10572 13.951 6.02262C13.7659 5.93952 13.5649 5.89767 13.3619 5.89997H9.39995ZM4.49998 14.9999H2.39999C2.02869 14.9999 1.6726 14.8524 1.41005 14.5899C1.1475 14.3273 1 13.9712 1 13.5999V8.69995C1 8.32865 1.1475 7.97256 1.41005 7.71001C1.6726 7.44746 2.02869 7.29996 2.39999 7.29996H4.49998" fill="none"/>
+<path d="M4.49998 7.29996L7.29996 1C7.85691 1 8.39105 1.22125 8.78487 1.61507C9.1787 2.0089 9.39995 2.54304 9.39995 3.09999V5.89997H13.3619C13.5649 5.89767 13.7659 5.93952 13.951 6.02262C14.1362 6.10572 14.301 6.22808 14.4342 6.38123C14.5674 6.53437 14.6657 6.71464 14.7223 6.90954C14.7788 7.10444 14.7924 7.30931 14.7619 7.50996L13.7959 13.8099C13.7453 14.1437 13.5757 14.448 13.3185 14.6667C13.0612 14.8854 12.7336 15.0037 12.3959 14.9999H4.49998M4.49998 7.29996V14.9999M4.49998 7.29996H2.39999C2.02869 7.29996 1.6726 7.44746 1.41005 7.71001C1.1475 7.97256 1 8.32865 1 8.69995V13.5999C1 13.9712 1.1475 14.3273 1.41005 14.5899C1.6726 14.8524 2.02869 14.9999 2.39999 14.9999H4.49998" stroke="current" stroke-width="1.39999" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

extensions/react-widget/src/assets/message.svg

@@ -1,7 +0,0 @@
-<svg width="36" height="36" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M4.37891 9.44824H7.75821" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M11.1377 9.44824H12.8273" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M4.37891 6.06934H6.06856" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M9.44824 6.06934H12.8276" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M16.2069 11.1379C16.2069 11.5861 16.0289 12.0158 15.712 12.3327C15.3951 12.6496 14.9654 12.8276 14.5172 12.8276H4.37931L1 16.2069V2.68965C1 2.24153 1.17802 1.81176 1.49489 1.49489C1.81176 1.17802 2.24153 1 2.68965 1H14.5172C14.9654 1 15.3951 1.17802 15.712 1.49489C16.0289 1.81176 16.2069 2.24153 16.2069 2.68965V11.1379Z" stroke="white" stroke-width="1.68965" stroke-linecap="round" stroke-linejoin="round"/>
-</svg>

extensions/react-widget/src/components/DocsGPTWidget.tsx

@@ -1,13 +1,40 @@
 "use client";
-import React from 'react'
+import React, { useRef } from 'react'
 import DOMPurify from 'dompurify';
-import snarkdown from '@bpmn-io/snarkdown';
 import styled, { keyframes, createGlobalStyle } from 'styled-components';
 import { PaperPlaneIcon, RocketIcon, ExclamationTriangleIcon, Cross2Icon } from '@radix-ui/react-icons';
-import MessageIcon from '../assets/message.svg';
-import { MESSAGE_TYPE, Query, Status } from '../types/index';
-import { fetchAnswerStreaming } from '../requests/streamingApi';
-
+import { FEEDBACK, MESSAGE_TYPE, Query, Status, WidgetProps } from '../types/index';
+import { fetchAnswerStreaming, sendFeedback } from '../requests/streamingApi';
+import { ThemeProvider } from 'styled-components';
+import Like from "../assets/like.svg"
+import Dislike from "../assets/dislike.svg"
+import MarkdownIt from 'markdown-it';
+const themes = {
+  dark: {
+    bg: '#222327',
+    text: '#fff',
+    primary: {
+      text: "#FAFAFA",
+      bg: '#222327'
+    },
+    secondary: {
+      text: "#A1A1AA",
+      bg: "#38383b"
+    }
+  },
+  light: {
+    bg: '#fff',
+    text: '#000',
+    primary: {
+      text: "#222327",
+      bg: "#fff"
+    },
+    secondary: {
+      text: "#A1A1AA",
+      bg: "#F6F6F6"
+    }
+  }
+}
 const GlobalStyles = createGlobalStyle`
 .response pre {
     padding: 8px;
@@ -16,6 +43,7 @@ const GlobalStyles = createGlobalStyle`
     border-radius: 6px;
     overflow-x: auto;
     background-color: #1B1C1F;
+    color: #fff !important;
 }
 .response h1{
   font-size: 20px;
@@ -26,40 +54,64 @@ const GlobalStyles = createGlobalStyle`
 .response h3{
   font-size: 16px;
 }
+.response p{
+  margin:0px;
+}
 .response code:not(pre code){
   border-radius: 6px;
   padding: 1px 3px 1px 3px;
   font-size: 12px;
   display: inline-block;
   background-color: #646464;
+  color: #fff !important;
+}
+.response code {
+  white-space: pre-wrap !important;
+  line-break: loose !important;
 }
 `;
-const WidgetContainer = styled.div`
+const Overlay = styled.div`
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background-color: rgba(0, 0, 0, 0.5);
+  z-index: 999;
+  transition: opacity 0.5s;
+`
+const WidgetContainer = styled.div<{ modal: boolean }>`
     display: block;
     position: fixed;
-    right: 10px;
-    bottom: 10px;
+    right: ${props => props.modal ? '50%' : '10px'};
+    bottom: ${props => props.modal ? '50%' : '10px'};
     z-index: 1000;
     display: flex;
+    ${props => props.modal &&
+    "transform : translate(50%,50%);"
+  }
     flex-direction: column;
     align-items: center;
     text-align: left;
+    @media only screen and (max-width: 768px) {
+    max-height: 100vh !important;
+    overflow: auto;
+    }
 `;
 const StyledContainer = styled.div`
-    display: block;
+    display: flex;
     position: relative;
+    flex-direction: column;
+    justify-content: center;
     bottom: 0;
     left: 0;
-    width: 352px;
-    height: 407px;
-    max-height: 407px;
     border-radius: 0.75rem;
-    background-color: #222327;
+    background-color: ${props => props.theme.primary.bg};
     font-family: sans-serif;
     box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05), 0 2px 4px rgba(0, 0, 0, 0.1);
     transition: visibility 0.3s, opacity 0.3s;
 `;
-const FloatingButton = styled.div`
+const FloatingButton = styled.div<{ bgcolor: string }>`
     position: fixed;
     display: flex;
     z-index: 500;
@@ -70,7 +122,7 @@ const FloatingButton = styled.div`
     width: 5rem;
     height: 5rem;
     border-radius: 9999px;
-    background-image: linear-gradient(to bottom right, #5AF0EC, #E80D9D);
+    background: ${props => props.bgcolor};
     box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
     cursor: pointer;
     &:hover {
@@ -120,39 +172,62 @@ const ContentWrapper = styled.div`
 const Title = styled.h3`
     font-size: 1rem;
     font-weight: normal;
-    color: #FAFAFA;
+    color: ${props => props.theme.primary.text};
     margin-top: 0;
     margin-bottom: 0.25rem;
 `;
 
 const Description = styled.p`
     font-size: 0.85rem;
-    color: #A1A1AA;
+    color: ${props => props.theme.secondary.text};
     margin-top: 0;
 `;
-const Conversation = styled.div`
-    height: 16rem;
+
+const Conversation = styled.div<{ size: string }>`
+    min-height: 250px;
+    max-width: 968px;
+    height: ${props => props.size === 'large' ? '75vh' : props.size === 'medium' ? '70vh' : '320px'};
+    width: ${props => props.size === 'large' ? '60vw' : props.size === 'medium' ? '28vw' : '400px'};
     padding-inline: 0.5rem;
     border-radius: 0.375rem;
     text-align: left;
     overflow-y: auto;
     scrollbar-width: thin;
     scrollbar-color: #4a4a4a transparent; /* thumb color track color */
+    @media only screen and (max-width: 768px) {
+    width: 90vw !important;
+    }
+    @media only screen and (min-width:768px ) and (max-width: 1280px) {
+    width:${props => props.size === 'large' ? '90vw' : props.size === 'medium' ? '60vw' : '400px'} !important;
+    }
+`;
+const Feedback = styled.div`
+  background-color: transparent;
+  font-weight: normal;
+  gap: 12px;
+  display: flex;
+  padding: 6px;
+  clear: both;
 `;
-
 const MessageBubble = styled.div<{ type: MESSAGE_TYPE }>`
-    display: flex;
+    display: block;
     font-size: 16px;
-    justify-content: ${props => props.type === 'QUESTION' ? 'flex-end' : 'flex-start'};
-    margin: 0.5rem;
+    position: relative;
+    width: 100%;;
+    float: right;
+    margin: 0rem;
+    &:hover ${Feedback} * {
+    visibility: visible !important;
+  }
 `;
-const Message = styled.p<{ type: MESSAGE_TYPE }>`
+const Message = styled.div<{ type: MESSAGE_TYPE }>`
     background: ${props => props.type === 'QUESTION' ?
     'linear-gradient(to bottom right, #8860DB, #6D42C5)' :
-    '#38383b'};
-    color: #ffff;
+    props.theme.secondary.bg};
+    color: ${props => props.type === 'ANSWER' ? props.theme.primary.text : '#fff'};
     border: none;
-    max-width: 80%;
+    float: ${props => props.type === 'QUESTION' ? 'right' : 'left'};
+    max-width: ${props => props.type === 'ANSWER' ? '100%' : '80'};
     overflow: auto;
     margin: 4px;
     display: block;
@@ -190,35 +265,31 @@ const DotAnimation = styled.div`
 const Delay = styled(DotAnimation) <{ delay: number }>`
   animation-delay: ${props => props.delay + 'ms'};
 `;
-const PromptContainer = styled.form`
+const PromptContainer = styled.form<{ size: string }>`
   background-color: transparent;
-  height: 36px;
-  position: absolute;
-  bottom: 25px;
-  left: 24px;
-  right: 24px;
+  height: ${props => props.size == 'large' ? '60px' : '40px'};
+  margin: 16px;
   display: flex;
   justify-content: space-evenly;
 `;
 const StyledInput = styled.input`
-  width: 260px;
-  height: 36px;
+  width: 100%;
   border: 1px solid #686877;
   padding-left: 12px;
   background-color: transparent;
   font-size: 16px;
   border-radius: 6px;
-  color: #ffff;
+  color: ${props => props.theme.text};
   outline: none;
 `;
-const StyledButton = styled.button`
+const StyledButton = styled.button<{ size: string }>`
   display: flex;
   justify-content: center;
   align-items: center;
   background-image: linear-gradient(to bottom right, #5AF0EC, #E80D9D);
   border-radius: 6px;
-  width: 36px;
-  height: 36px;
+  min-width: ${props => props.size === 'large' ? '60px' : '36px'};
+  height: ${props => props.size === 'large' ? '60px' : '36px'};
   margin-left:8px;
   padding: 0px;
   border: none;
@@ -239,13 +310,14 @@ const HeroContainer = styled.div`
   align-items: middle;
   transform: translate(-50%, -50%);
   width: 80%;
+  max-width: 500px;
   background-image: linear-gradient(to bottom right, #5AF0EC, #ff1bf4);
   border-radius: 10px;
   margin: 0 auto;
   padding: 2px;
 `;
 const HeroWrapper = styled.div`
-  background-color: #222327;
+  background-color: ${props => props.theme.primary.bg};
   border-radius: 10px; 
   font-weight: normal;
   padding: 6px;
@@ -253,23 +325,23 @@ const HeroWrapper = styled.div`
   justify-content: space-between;
 `
 const HeroTitle = styled.h3`
-  color: #fff;
-  font-size: 17px;
+  color: ${props => props.theme.text};
   margin-bottom: 5px;
   padding: 2px;
 `;
 const HeroDescription = styled.p`
-  color: #fff;
+  color: ${props => props.theme.text};
   font-size: 14px;
   line-height: 1.5;
 `;
-const Hero = ({ title, description }: { title: string, description: string }) => {
+
+const Hero = ({ title, description, theme }: { title: string, description: string, theme: string }) => {
   return (
     <>
       <HeroContainer>
         <HeroWrapper>
-          <IconWrapper style={{ marginTop: '8px' }}>
-            <RocketIcon color='white' width={20} height={20} />
+          <IconWrapper style={{ marginTop: '12px' }}>
+            <RocketIcon color={theme === 'light' ? 'black' : 'white'} width={20} height={20} />
           </IconWrapper>
           <div>
             <HeroTitle>{title}</HeroTitle>
@@ -284,22 +356,28 @@ const Hero = ({ title, description }: { title: string, description: string }) =>
 };
 export const DocsGPTWidget = ({
   apiHost = 'https://gptcloud.arc53.com',
-  selectDocs = 'default',
   apiKey = '82962c9a-aa77-4152-94e5-a4f84fd44c6a',
   avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
   title = 'Get AI assistance',
   description = 'DocsGPT\'s AI Chatbot is here to help',
   heroTitle = 'Welcome to DocsGPT !',
-  heroDescription = 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.'
-}) => {
-
+  heroDescription = 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.',
+  size = 'small',
+  theme = 'dark',
+  buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/message.svg',
+  buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)',
+  collectFeedback = true
+}: WidgetProps) => {
   const [prompt, setPrompt] = React.useState('');
   const [status, setStatus] = React.useState<Status>('idle');
   const [queries, setQueries] = React.useState<Query[]>([])
   const [conversationId, setConversationId] = React.useState<string | null>(null)
   const [open, setOpen] = React.useState<boolean>(false)
   const [eventInterrupt, setEventInterrupt] = React.useState<boolean>(false); //click or scroll by user while autoScrolling
+  const isBubbleHovered = useRef<boolean>(false)
   const endMessageRef = React.useRef<HTMLDivElement | null>(null);
+  const md = new MarkdownIt();
+
   const handleUserInterrupt = () => {
     (status === 'loading') && setEventInterrupt(true);
   }
@@ -316,11 +394,40 @@ export const DocsGPTWidget = ({
     const lastChild = element?.children?.[element.children.length - 1]
     lastChild && scrollToBottom(lastChild)
   };
-
   React.useEffect(() => {
     !eventInterrupt && scrollToBottom(endMessageRef.current);
   }, [queries.length, queries[queries.length - 1]?.response]);
 
+  async function handleFeedback(feedback: FEEDBACK, index: number) {
+    let query = queries[index]
+    if (!query.response)
+      return;
+    if (query.feedback != feedback) {
+      sendFeedback({
+        question: query.prompt,
+        answer: query.response,
+        feedback: feedback,
+        apikey: apiKey
+      }, apiHost)
+        .then(res => {
+          if (res.status == 200) {
+            query.feedback = feedback;
+            setQueries((prev: Query[]) => {
+              return prev.map((q, i) => (i === index ? query : q));
+            });
+          }
+        })
+        .catch(err => console.log("Connection failed",err))
+    }
+    else {
+      delete query.feedback;
+      setQueries((prev: Query[]) => {
+        return prev.map((q, i) => (i === index ? query : q));
+      });
+
+    }
+  }
+
   async function stream(question: string) {
     setStatus('loading')
     try {
@@ -329,19 +436,24 @@ export const DocsGPTWidget = ({
           question: question,
           apiKey: apiKey,
           apiHost: apiHost,
-          selectedDocs: selectDocs,
           history: queries,
           conversationId: conversationId,
           onEvent: (event: MessageEvent) => {
             const data = JSON.parse(event.data);
             // check if the 'end' event has been received
             if (data.type === 'end') {
-              // set status to 'idle'
               setStatus('idle');
-
-            } else if (data.type === 'id') {
+            }
+            else if (data.type === 'id') {
               setConversationId(data.id)
-            } else {
+            }
+            else if (data.type === 'error') {
+              const updatedQueries = [...queries];
+              updatedQueries[updatedQueries.length - 1].error = data.error;
+              setQueries(updatedQueries);
+              setStatus('idle')
+            }
+            else {
               const result = data.answer;
               const streamingResponse = queries[queries.length - 1].response ? queries[queries.length - 1].response : '';
               const updatedQueries = [...queries];
@@ -353,7 +465,7 @@ export const DocsGPTWidget = ({
       );
     } catch (error) {
       const updatedQueries = [...queries];
-      updatedQueries[updatedQueries.length - 1].error = 'error'
+      updatedQueries[updatedQueries.length - 1].error = 'Something went wrong !'
       setQueries(updatedQueries);
       setStatus('idle')
       //setEventInterrupt(false)
@@ -372,16 +484,21 @@ export const DocsGPTWidget = ({
     event.currentTarget.src = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png";
   };
   return (
-    <>
-      <WidgetContainer>
+    <ThemeProvider theme={themes[theme]}>
+      {open && size === 'large' &&
+        <Overlay onClick={() => {
+          setOpen(false)
+        }} />
+      }
+      <FloatingButton bgcolor={buttonBg} onClick={() => setOpen(!open)} hidden={open}>
+        <img style={{ maxHeight: '4rem', maxWidth: '4rem' }} src={buttonIcon} />
+      </FloatingButton>
+      <WidgetContainer modal={size == 'large'}>
         <GlobalStyles />
-        {!open && <FloatingButton onClick={() => setOpen(true)} hidden={open}>
-          <MessageIcon style={{ marginTop: '8px' }} />
-        </FloatingButton>}
         {open && <StyledContainer>
           <div>
             <CancelButton onClick={() => setOpen(false)}>
-              <Cross2Icon width={24} height={24} color='white' />
+              <Cross2Icon width={24} height={24} color={theme === 'light' ? 'black' : 'white'} />
             </CancelButton>
             <Header>
               <IconWrapper>
@@ -393,7 +510,7 @@ export const DocsGPTWidget = ({
               </ContentWrapper>
             </Header>
           </div>
-          <Conversation onWheel={handleUserInterrupt} onTouchMove={handleUserInterrupt}>
+          <Conversation size={size} onWheel={handleUserInterrupt} onTouchMove={handleUserInterrupt}>
             {
               queries.length > 0 ? queries?.map((query, index) => {
                 return (
@@ -408,13 +525,34 @@ export const DocsGPTWidget = ({
                       </MessageBubble>
                     }
                     {
-                      query.response ? <MessageBubble type='ANSWER'>
+                      query.response ? <MessageBubble onMouseOver={() => { isBubbleHovered.current = true }} type='ANSWER'>
                         <Message
                           type='ANSWER'
                           ref={(index === queries.length - 1) ? endMessageRef : null}
                         >
-                          <div className="response" dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(snarkdown(query.response)) }} />
+                          <div
+                            className="response"
+                            dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(md.render(query.response)) }}
+                          />
                         </Message>
+
+                        {collectFeedback &&
+                          <Feedback>
+                            <Like
+                              style={{
+                                stroke: query.feedback == 'LIKE' ? '#8860DB' : '#c0c0c0',
+                                visibility: query.feedback == 'LIKE' ? 'visible' : 'hidden'
+                              }}
+                              fill='none'
+                              onClick={() => handleFeedback("LIKE", index)} />
+                            <Dislike
+                              style={{
+                                stroke: query.feedback == 'DISLIKE' ? '#ed8085' : '#c0c0c0',
+                                visibility: query.feedback == 'DISLIKE' ? 'visible' : 'hidden'
+                              }}
+                              fill='none'
+                              onClick={() => handleFeedback("DISLIKE", index)} />
+                          </Feedback>}
                       </MessageBubble>
                         : <div>
                           {
@@ -424,7 +562,7 @@ export const DocsGPTWidget = ({
                               </IconWrapper>
                               <div>
                                 <h5 style={{ margin: 2 }}>Network Error</h5>
-                                <span style={{ margin: 2, fontSize: '13px' }}>Something went wrong !</span>
+                                <span style={{ margin: 2, fontSize: '13px' }}>{query.error}</span>
                               </div>
                             </ErrorAlert>
                               : <MessageBubble type='ANSWER'>
@@ -439,22 +577,23 @@ export const DocsGPTWidget = ({
                     }
                   </React.Fragment>)
               })
-                : <Hero title={heroTitle} description={heroDescription} />
+                : <Hero title={heroTitle} description={heroDescription} theme={theme} />
             }
           </Conversation>
-
           <PromptContainer
+            size={size}
             onSubmit={handleSubmit}>
             <StyledInput
               value={prompt} onChange={(event) => setPrompt(event.target.value)}
               type='text' placeholder="What do you want to do?" />
             <StyledButton
-              disabled={prompt.length == 0 || status !== 'idle'}>
+              size={size}
+              disabled={prompt.trim().length == 0 || status !== 'idle'}>
               <PaperPlaneIcon width={15} height={15} color='white' />
             </StyledButton>
           </PromptContainer>
         </StyledContainer>}
       </WidgetContainer>
-    </>
+    </ThemeProvider>
   )
 }

extensions/react-widget/src/main.tsx

@@ -2,10 +2,11 @@ import React from 'react';
 import { createRoot } from 'react-dom/client';
 import { DocsGPTWidget } from './components/DocsGPTWidget';
 
-const renderWidget = (elementId: string, props = {}) => {
-  const root = createRoot(document.getElementById(elementId) as HTMLElement);
-  root.render(<DocsGPTWidget {...props} />);
-};
-
-(window as any).renderDocsGPTWidget = renderWidget;
+if (typeof window !== 'undefined') {
+  const renderWidget = (elementId: string, props = {}) => {
+    const root = createRoot(document.getElementById(elementId) as HTMLElement);
+    root.render(<DocsGPTWidget {...props} />);
+  };
+  (window as any).renderDocsGPTWidget = renderWidget;
+}
 export { DocsGPTWidget };

extensions/react-widget/src/requests/streamingApi.ts

@@ -1,92 +1,106 @@
+import { FEEDBACK } from "@/types";
 interface HistoryItem {
-    prompt: string;
-    response?: string;
-  }
+  prompt: string;
+  response?: string;
+}
 interface FetchAnswerStreamingProps {
-    question?: string;
-    apiKey?: string;
-    selectedDocs?: string;
-    history?: HistoryItem[];
-    conversationId?: string | null;
-    apiHost?: string;
-    onEvent?: (event: MessageEvent) => void;
-  }
+  question?: string;
+  apiKey?: string;
+  selectedDocs?: string;
+  history?: HistoryItem[];
+  conversationId?: string | null;
+  apiHost?: string;
+  onEvent?: (event: MessageEvent) => void;
+}
+interface FeedbackPayload {
+  question: string;
+  answer: string;
+  apikey: string;
+  feedback: FEEDBACK;
+}
 export function fetchAnswerStreaming({
-    question = '',
-    apiKey = '',
-    selectedDocs = '',
-    history = [],
-    conversationId = null,
-    apiHost = '',
-    onEvent = () => {console.log("Event triggered, but no handler provided.");}
-  }: FetchAnswerStreamingProps): Promise<void> {
-    let docPath = 'default';
-    if (selectedDocs) {
-      docPath = selectedDocs;
-    }
-  
-    return new Promise<void>((resolve, reject) => {
-      const body = {
-        question: question,
-        api_key: apiKey,
-        embeddings_key: apiKey,
-        active_docs: docPath,
-        history: JSON.stringify(history),
-        conversation_id: conversationId,
-        model: 'default'
-      };
-  
-      fetch(apiHost + '/stream', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify(body),
-      })
-        .then((response) => {
-          if (!response.body) throw Error('No response body');
-  
-          const reader = response.body.getReader();
-          const decoder = new TextDecoder('utf-8');
-          let counterrr = 0;
-          const processStream = ({
-            done,
-            value,
-          }: ReadableStreamReadResult<Uint8Array>) => {
-            if (done) {
-              resolve();
-              return;
+  question = '',
+  apiKey = '',
+  history = [],
+  conversationId = null,
+  apiHost = '',
+  onEvent = () => { console.log("Event triggered, but no handler provided."); }
+}: FetchAnswerStreamingProps): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const body = {
+      question: question,
+      history: JSON.stringify(history),
+      conversation_id: conversationId,
+      model: 'default',
+      api_key: apiKey
+    };
+    fetch(apiHost + '/stream', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(body),
+    })
+      .then((response) => {
+        if (!response.body) throw Error('No response body');
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder('utf-8');
+        let counterrr = 0;
+        const processStream = ({
+          done,
+          value,
+        }: ReadableStreamReadResult<Uint8Array>) => {
+          if (done) {
+            resolve();
+            return;
+          }
+
+          counterrr += 1;
+
+          const chunk = decoder.decode(value);
+
+          const lines = chunk.split('\n');
+
+          for (let line of lines) {
+            if (line.trim() == '') {
+              continue;
             }
-  
-            counterrr += 1;
-  
-            const chunk = decoder.decode(value);
-  
-            const lines = chunk.split('\n');
-  
-            for (let line of lines) {
-              if (line.trim() == '') {
-                continue;
-              }
-              if (line.startsWith('data:')) {
-                line = line.substring(5);
-              }
-  
-              const messageEvent = new MessageEvent('message', {
-                data: line,
-              });
-  
-              onEvent(messageEvent); // handle each message
+            if (line.startsWith('data:')) {
+              line = line.substring(5);
             }
-  
-            reader.read().then(processStream).catch(reject);
-          };
-  
+
+            const messageEvent = new MessageEvent('message', {
+              data: line,
+            });
+
+            onEvent(messageEvent); // handle each message
+          }
+
           reader.read().then(processStream).catch(reject);
-        })
-        .catch((error) => {
-          console.error('Connection failed:', error);
-          reject(error);
-        });
-    });
-  }
+        };
+
+        reader.read().then(processStream).catch(reject);
+      })
+      .catch((error) => {
+        console.error('Connection failed:', error);
+        reject(error);
+      });
+  });
+}
+
+
+export const sendFeedback = (payload: FeedbackPayload,apiHost:string): Promise<Response> => {
+  return fetch(`${apiHost}/api/feedback`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      question: payload.question,
+      answer: payload.answer,
+      feedback: payload.feedback,
+      api_key:payload.apikey
+    }),
+  });
+};

extensions/react-widget/src/types/index.ts

@@ -1,7 +1,7 @@
 export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER' | 'ERROR';
 export type Status = 'idle' | 'loading' | 'failed';
 export type FEEDBACK = 'LIKE' | 'DISLIKE';
-
+export type THEME = 'light' | 'dark';
 export interface Query {
   prompt: string;
   response?: string;
@@ -10,4 +10,18 @@ export interface Query {
   sources?: { title: string; text: string }[];
   conversationId?: string | null;
   title?: string | null;
+}
+export interface WidgetProps {
+  apiHost?: string;
+  apiKey?: string;
+  avatar?: string;
+  title?: string;
+  description?: string;
+  heroTitle?: string;
+  heroDescription?: string;
+  size?: 'small' | 'medium' | 'large';
+  theme?:THEME,
+  buttonIcon?:string;
+  buttonBg?:string;
+  collectFeedback?:boolean
 }

frontend/package.json

@@ -21,15 +21,17 @@
   "dependencies": {
     "@reduxjs/toolkit": "^2.2.7",
     "@vercel/analytics": "^0.1.10",
+    "chart.js": "^4.4.4",
     "i18next": "^23.14.0",
     "i18next-browser-languagedetector": "^8.0.0",
     "prop-types": "^15.8.1",
     "react": "^18.2.0",
+    "react-chartjs-2": "^5.2.0",
     "react-copy-to-clipboard": "^5.1.0",
     "react-dom": "^18.3.1",
     "react-dropzone": "^14.2.3",
     "react-i18next": "^15.0.1",
-    "react-markdown": "^8.0.7",
+    "react-markdown": "^9.0.1",
     "react-redux": "^8.0.5",
     "react-router-dom": "^6.8.1",
     "react-syntax-highlighter": "^15.5.0",
@@ -38,17 +40,17 @@
   "devDependencies": {
     "@types/react": "^18.0.27",
     "@types/react-dom": "^18.3.0",
-    "@types/react-syntax-highlighter": "^15.5.6",
+    "@types/react-syntax-highlighter": "^15.5.13",
     "@typescript-eslint/eslint-plugin": "^5.51.0",
     "@typescript-eslint/parser": "^5.62.0",
     "@vitejs/plugin-react": "^4.3.1",
     "autoprefixer": "^10.4.13",
-    "eslint": "^8.33.0",
-    "eslint-config-prettier": "^8.6.0",
+    "eslint": "^8.57.1",
+    "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard-with-typescript": "^34.0.0",
     "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-n": "^15.7.0",
-    "eslint-plugin-prettier": "^4.2.1",
+    "eslint-plugin-prettier": "^5.2.1",
     "eslint-plugin-promise": "^6.6.0",
     "eslint-plugin-react": "^7.35.0",
     "eslint-plugin-unused-imports": "^2.0.0",
@@ -56,10 +58,10 @@
     "lint-staged": "^15.2.8",
     "postcss": "^8.4.41",
     "prettier": "^3.3.3",
-    "prettier-plugin-tailwindcss": "^0.2.2",
-    "tailwindcss": "^3.2.4",
+    "prettier-plugin-tailwindcss": "^0.6.6",
+    "tailwindcss": "^3.4.11",
     "typescript": "^4.9.5",
-    "vite": "^5.3.5",
+    "vite": "^5.4.6",
     "vite-plugin-svgr": "^4.2.0"
   }
 }

frontend/src/Navigation.tsx

@@ -24,9 +24,9 @@ import ConversationTile from './conversation/ConversationTile';
 import { useDarkTheme, useMediaQuery, useOutsideAlerter } from './hooks';
 import useDefaultDocument from './hooks/useDefaultDocument';
 import DeleteConvModal from './modals/DeleteConvModal';
-import { ActiveState } from './models/misc';
+import { ActiveState, Doc } from './models/misc';
 import APIKeyModal from './preferences/APIKeyModal';
-import { Doc, getConversations, getDocs } from './preferences/preferenceApi';
+import { getConversations, getDocs } from './preferences/preferenceApi';
 import {
   selectApiKeyStatus,
   selectConversationId,
@@ -124,10 +124,8 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   };
 
   const handleDeleteClick = (doc: Doc) => {
-    const docPath = `indexes/local/${doc.name}`;
-
     userService
-      .deletePath(docPath)
+      .deletePath(doc.id ?? '')
       .then(() => {
         return getDocs();
       })
@@ -174,16 +172,12 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         console.error(err);
       });
   }
-  useOutsideAlerter(
-    navRef,
-    () => {
-      if (isMobile && navOpen && apiKeyModalState === 'INACTIVE') {
-        setNavOpen(false);
-        setIsDocsListOpen(false);
-      }
-    },
-    [navOpen, isDocsListOpen, apiKeyModalState],
-  );
+  useOutsideAlerter(navRef, () => {
+    if (isMobile && navOpen && apiKeyModalState === 'INACTIVE') {
+      setNavOpen(false);
+      setIsDocsListOpen(false);
+    }
+  }, [navOpen, isDocsListOpen, apiKeyModalState]);
 
   /*
     Needed to fix bug where if mobile nav was closed and then window was resized to desktop, nav would still be closed but the button to open would be gone, as per #1 on issue #146

frontend/src/api/endpoints.ts

@@ -10,8 +10,12 @@ const endpoints = {
     DELETE_PROMPT: '/api/delete_prompt',
     UPDATE_PROMPT: '/api/update_prompt',
     SINGLE_PROMPT: (id: string) => `/api/get_single_prompt?id=${id}`,
-    DELETE_PATH: (docPath: string) => `/api/delete_old?path=${docPath}`,
+    DELETE_PATH: (docPath: string) => `/api/delete_old?source_id=${docPath}`,
     TASK_STATUS: (task_id: string) => `/api/task_status?task_id=${task_id}`,
+    MESSAGE_ANALYTICS: '/api/get_message_analytics',
+    TOKEN_ANALYTICS: '/api/get_token_analytics',
+    FEEDBACK_ANALYTICS: '/api/get_feedback_analytics',
+    LOGS: `/api/get_user_logs`,
   },
   CONVERSATION: {
     ANSWER: '/api/answer',

frontend/src/api/services/userService.ts

@@ -23,6 +23,14 @@ const userService = {
     apiClient.get(endpoints.USER.DELETE_PATH(docPath)),
   getTaskStatus: (task_id: string): Promise<any> =>
     apiClient.get(endpoints.USER.TASK_STATUS(task_id)),
+  getMessageAnalytics: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.MESSAGE_ANALYTICS, data),
+  getTokenAnalytics: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.TOKEN_ANALYTICS, data),
+  getFeedbackAnalytics: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.FEEDBACK_ANALYTICS, data),
+  getLogs: (data: any): Promise<any> =>
+    apiClient.post(endpoints.USER.LOGS, data),
 };
 
 export default userService;

frontend/src/assets/chevron-right.svg

@@ -0,0 +1,3 @@
+<svg width="7" height="12" viewBox="0 0 7 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.29154 4.88202L1.70154 0.29202C1.60896 0.199438 1.49905 0.125998 1.37808 0.0758932C1.25712 0.0257882 1.12747 -2.37536e-07 0.99654 -2.44235e-07C0.86561 -2.50934e-07 0.735961 0.0257882 0.614997 0.0758931C0.494033 0.125998 0.384122 0.199438 0.29154 0.29202C0.198958 0.384602 0.125519 0.494513 0.0754137 0.615477C0.0253086 0.736441 -0.00048069 0.86609 -0.000480695 0.99702C-0.000480701 1.12795 0.0253086 1.2576 0.0754136 1.37856C0.125519 1.49953 0.198958 1.60944 0.29154 1.70202L4.17154 5.59202L0.29154 9.47202C0.198958 9.5646 0.125518 9.67451 0.0754133 9.79548C0.0253082 9.91644 -0.000481091 10.0461 -0.000481097 10.177C-0.000481102 10.3079 0.0253082 10.4376 0.0754132 10.5586C0.125518 10.6795 0.198958 10.7894 0.29154 10.882C0.384121 10.9746 0.494032 11.048 0.614996 11.0981C0.73596 11.1483 0.865609 11.174 0.99654 11.174C1.12747 11.174 1.25712 11.1483 1.37808 11.0981C1.49905 11.048 1.60896 10.9746 1.70154 10.882L6.29154 6.29202C6.38424 6.19951 6.45779 6.08962 6.50797 5.96864C6.55815 5.84767 6.58398 5.71799 6.58398 5.58702C6.58398 5.45605 6.55815 5.32637 6.50797 5.2054C6.45779 5.08442 6.38424 4.97453 6.29154 4.88202Z" fill="#666666"/>
+</svg>

frontend/src/components/CopyButton.tsx

@@ -1,16 +1,24 @@
-import { useState } from 'react';
-import Copy from './../assets/copy.svg?react';
-import CheckMark from './../assets/checkmark.svg?react';
 import copy from 'copy-to-clipboard';
+import { useState } from 'react';
 
-export default function CoppyButton({ text }: { text: string }) {
+import CheckMark from '../assets/checkmark.svg?react';
+import Copy from '../assets/copy.svg?react';
+
+export default function CoppyButton({
+  text,
+  colorLight,
+  colorDark,
+}: {
+  text: string;
+  colorLight?: string;
+  colorDark?: string;
+}) {
   const [copied, setCopied] = useState(false);
   const [isCopyHovered, setIsCopyHovered] = useState(false);
 
   const handleCopyClick = (text: string) => {
     copy(text);
     setCopied(true);
-    // Reset copied to false after a few seconds
     setTimeout(() => {
       setCopied(false);
     }, 3000);
@@ -20,8 +28,8 @@ export default function CoppyButton({ text }: { text: string }) {
     <div
       className={`flex items-center justify-center rounded-full p-2 ${
         isCopyHovered
-          ? 'bg-[#EEEEEE] dark:bg-purple-taupe'
-          : 'bg-[#ffffff] dark:bg-transparent'
+          ? `bg-[#EEEEEE] dark:bg-purple-taupe`
+          : `bg-[${colorLight ? colorLight : '#FFFFFF'}] dark:bg-[${colorDark ? colorDark : 'transparent'}]`
       }`}
     >
       {copied ? (

frontend/src/components/Dropdown.tsx

@@ -16,6 +16,7 @@ function Dropdown({
   showDelete,
   onDelete,
   placeholder,
+  contentSize = 'text-base',
 }: {
   options:
     | string[]
@@ -26,6 +27,7 @@ function Dropdown({
     | string
     | { label: string; value: string }
     | { value: number; description: string }
+    | { name: string; id: string; type: string }
     | null;
   onSelect:
     | ((value: string) => void)
@@ -41,6 +43,7 @@ function Dropdown({
   showDelete?: boolean;
   onDelete?: (value: string) => void;
   placeholder?: string;
+  contentSize?: string;
 }) {
   const dropdownRef = React.useRef<HTMLDivElement>(null);
   const [isOpen, setIsOpen] = React.useState(false);
@@ -84,21 +87,21 @@ function Dropdown({
           </span>
         ) : (
           <span
-            className={`overflow-hidden text-ellipsis dark:text-bright-gray ${
+            className={`truncate overflow-hidden dark:text-bright-gray ${
               !selectedValue && 'text-silver dark:text-gray-400'
-            }`}
+            } ${contentSize}`}
           >
             {selectedValue && 'label' in selectedValue
               ? selectedValue.label
               : selectedValue && 'description' in selectedValue
-              ? `${
-                  selectedValue.value < 1e9
-                    ? selectedValue.value + ` (${selectedValue.description})`
-                    : selectedValue.description
-                }`
-              : placeholder
-              ? placeholder
-              : 'From URL'}
+                ? `${
+                    selectedValue.value < 1e9
+                      ? selectedValue.value + ` (${selectedValue.description})`
+                      : selectedValue.description
+                  }`
+                : placeholder
+                  ? placeholder
+                  : 'From URL'}
           </span>
         )}
         <img
@@ -123,19 +126,19 @@ function Dropdown({
                   onSelect(option);
                   setIsOpen(false);
                 }}
-                className="ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3 dark:text-light-gray"
+                className={`ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3 dark:text-light-gray ${contentSize}`}
               >
                 {typeof option === 'string'
                   ? option
                   : option.name
-                  ? option.name
-                  : option.label
-                  ? option.label
-                  : `${
-                      option.value < 1e9
-                        ? option.value + ` (${option.description})`
-                        : option.description
-                    }`}
+                    ? option.name
+                    : option.label
+                      ? option.label
+                      : `${
+                          option.value < 1e9
+                            ? option.value + ` (${option.description})`
+                            : option.description
+                        }`}
               </span>
               {showEdit && onEdit && (
                 <img

frontend/src/components/Input.tsx

@@ -10,6 +10,7 @@ const Input = ({
   maxLength,
   className,
   colorVariant = 'silver',
+  borderVariant = 'thick',
   children,
   onChange,
   onPaste,
@@ -20,10 +21,13 @@ const Input = ({
     jet: 'border-jet',
     gray: 'border-gray-5000 dark:text-silver',
   };
-
+  const borderStyles = {
+    thin: 'border',
+    thick: 'border-2',
+  };
   return (
     <input
-      className={`h-[42px] w-full rounded-full border-2 px-3 outline-none dark:bg-transparent dark:text-white ${className} ${colorStyles[colorVariant]}`}
+      className={`h-[42px] w-full rounded-full px-3 py-1 outline-none dark:bg-transparent dark:text-white ${className} ${colorStyles[colorVariant]} ${borderStyles[borderVariant]}`}
       type={type}
       id={id}
       name={name}

frontend/src/components/SourceDropdown.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import Trash from '../assets/trash.svg';
 import Arrow2 from '../assets/dropdown-arrow.svg';
-import { Doc } from '../preferences/preferenceApi';
+import { Doc } from '../models/misc';
 import { useDispatch } from 'react-redux';
 import { useTranslation } from 'react-i18next';
 type Props = {
@@ -63,9 +63,6 @@ function SourceDropdown({
             <p className="max-w-3/4 truncate whitespace-nowrap">
               {selectedDocs?.name || 'None'}
             </p>
-            <p className="flex flex-col items-center justify-center">
-              {selectedDocs?.version}
-            </p>
           </div>
         </span>
         <img

frontend/src/components/types/index.ts

@@ -2,6 +2,7 @@ export type InputProps = {
   type: 'text' | 'number';
   value: string | string[] | number;
   colorVariant?: 'silver' | 'jet' | 'gray';
+  borderVariant?: 'thin' | 'thick';
   isAutoFocused?: boolean;
   id?: string;
   maxLength?: number;

frontend/src/conversation/Conversation.tsx

@@ -1,4 +1,4 @@
-import React, { Fragment, useEffect, useRef, useState } from 'react';
+import { Fragment, useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
 
@@ -30,7 +30,7 @@ export default function Conversation() {
   const status = useSelector(selectStatus);
   const conversationId = useSelector(selectConversationId);
   const dispatch = useDispatch<AppDispatch>();
-  const endMessageRef = useRef<HTMLDivElement>(null);
+  const conversationRef = useRef<HTMLDivElement>(null);
   const inputRef = useRef<HTMLTextAreaElement>(null);
   const [isDarkTheme] = useDarkTheme();
   const [hasScrolledToLast, setHasScrolledToLast] = useState(true);
@@ -55,32 +55,8 @@ export default function Conversation() {
   }, []);
 
   useEffect(() => {
-    return () => {
-      if (status !== 'idle') {
-        fetchStream.current && fetchStream.current.abort(); //abort previous stream
-      }
-    };
-  }, [status]);
-
-  useEffect(() => {
-    const observerCallback: IntersectionObserverCallback = (entries) => {
-      entries.forEach((entry) => {
-        setHasScrolledToLast(entry.isIntersecting);
-      });
-    };
-
-    const observer = new IntersectionObserver(observerCallback, {
-      root: null,
-      threshold: [1, 0.8],
-    });
-    if (endMessageRef.current) {
-      observer.observe(endMessageRef.current);
-    }
-
-    return () => {
-      observer.disconnect();
-    };
-  }, [endMessageRef.current]);
+    fetchStream.current && fetchStream.current.abort();
+  }, [conversationId]);
 
   useEffect(() => {
     if (queries.length) {
@@ -90,10 +66,16 @@ export default function Conversation() {
   }, [queries[queries.length - 1]]);
 
   const scrollIntoView = () => {
-    endMessageRef?.current?.scrollIntoView({
-      behavior: 'smooth',
-      block: 'start',
-    });
+    if (!conversationRef?.current || eventInterrupt) return;
+
+    if (status === 'idle' || !queries[queries.length - 1].response) {
+      conversationRef.current.scrollTo({
+        behavior: 'smooth',
+        top: conversationRef.current.scrollHeight,
+      });
+    } else {
+      conversationRef.current.scrollTop = conversationRef.current.scrollHeight;
+    }
   };
 
   const handleQuestion = ({
@@ -147,7 +129,6 @@ export default function Conversation() {
     if (query.response) {
       responseView = (
         <ConversationBubble
-          ref={endMessageRef}
           className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'}`}
           key={`${index}ANSWER`}
           message={query.response}
@@ -180,7 +161,6 @@ export default function Conversation() {
       );
       responseView = (
         <ConversationBubble
-          ref={endMessageRef}
           className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'} `}
           key={`${index}ERROR`}
           message={query.error}
@@ -238,6 +218,7 @@ export default function Conversation() {
         </>
       )}
       <div
+        ref={conversationRef}
         onWheel={handleUserInterruption}
         onTouchMove={handleUserInterruption}
         className="flex h-[90%] w-full flex-1 justify-center overflow-y-auto p-4 md:h-[83vh]"

frontend/src/conversation/ConversationBubble.tsx

@@ -51,12 +51,13 @@ const ConversationBubble = forwardRef<
   let bubble;
   if (type === 'QUESTION') {
     bubble = (
-      <div ref={ref} className={`flex flex-row-reverse self-end ${className}`}>
+      <div
+        ref={ref}
+        className={`flex flex-row-reverse self-end flex-wrap ${className}`}
+      >
         <Avatar className="mt-2 text-2xl" avatar="🧑‍💻"></Avatar>
-        <div className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white">
-          <ReactMarkdown className="whitespace-pre-wrap break-normal leading-normal">
-            {message}
-          </ReactMarkdown>
+        <div className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white max-w-full whitespace-pre-wrap leading-normal break-normal">
+          {message}
         </div>
       </div>
     );
@@ -180,7 +181,7 @@ const ConversationBubble = forwardRef<
                   ))}
                   {(sources?.length ?? 0) > 3 && (
                     <div
-                      className="flex h-24 cursor-pointer flex-col-reverse rounded-[20px] bg-gray-1000 p-4 text-purple-30 hover:bg-[#F1F1F1] hover:text-[#6D3ECC] dark:bg-gun-metal dark:hover:bg-[#2C2E3C] dark:hover:text-[#8C67D7]"
+                      className="flex h-28 cursor-pointer flex-col-reverse rounded-[20px] bg-gray-1000 p-4 text-purple-30 hover:bg-[#F1F1F1] hover:text-[#6D3ECC] dark:bg-gun-metal dark:hover:bg-[#2C2E3C] dark:hover:text-[#8C67D7]"
                       onClick={() => setIsSidebarOpen(true)}
                     >
                       <p className="ellipsis-text h-22 text-xs">{`View ${
@@ -226,15 +227,16 @@ const ConversationBubble = forwardRef<
               className="whitespace-pre-wrap break-normal leading-normal"
               remarkPlugins={[remarkGfm]}
               components={{
-                code({ node, inline, className, children, ...props }) {
+                code(props) {
+                  const { children, className, node, ref, ...rest } = props;
                   const match = /language-(\w+)/.exec(className || '');
 
-                  return !inline && match ? (
+                  return match ? (
                     <div className="group relative">
                       <SyntaxHighlighter
+                        {...rest}
                         PreTag="div"
                         language={match[1]}
-                        {...props}
                         style={vscDarkPlus}
                       >
                         {String(children).replace(/\n$/, '')}
@@ -249,7 +251,10 @@ const ConversationBubble = forwardRef<
                       </div>
                     </div>
                   ) : (
-                    <code className={className ? className : ''} {...props}>
+                    <code
+                      className={className ? className : 'whitespace-pre-line'}
+                      {...props}
+                    >
                       {children}
                     </code>
                   );
@@ -396,8 +401,9 @@ const ConversationBubble = forwardRef<
             toggleState={(state: boolean) => {
               setIsSidebarOpen(state);
             }}
-            children={<AllSources sources={sources} />}
-          />
+          >
+            <AllSources sources={sources} />
+          </Sidebar>
         )}
       </div>
     );

frontend/src/conversation/SharedConversation.tsx

@@ -144,6 +144,7 @@ export const SharedConversation = () => {
           key={`${index}ANSWER`}
           message={query.response}
           type={'ANSWER'}
+          sources={query.sources ?? []}
         ></ConversationBubble>
       );
     } else if (query.error) {

frontend/src/conversation/conversationHandlers.ts

@@ -1,32 +1,6 @@
 import conversationService from '../api/services/conversationService';
-import { Doc } from '../preferences/preferenceApi';
-import { Answer, FEEDBACK } from './conversationModels';
-
-function getDocPath(selectedDocs: Doc | null): string {
-  let docPath = 'default';
-  if (selectedDocs) {
-    let namePath = selectedDocs.name;
-    if (selectedDocs.language === namePath) {
-      namePath = '.project';
-    }
-    if (selectedDocs.location === 'local') {
-      docPath = 'local' + '/' + selectedDocs.name + '/';
-    } else if (selectedDocs.location === 'remote') {
-      docPath =
-        selectedDocs.language +
-        '/' +
-        namePath +
-        '/' +
-        selectedDocs.version +
-        '/' +
-        selectedDocs.model +
-        '/';
-    } else if (selectedDocs.location === 'custom') {
-      docPath = selectedDocs.docLink;
-    }
-  }
-  return docPath;
-}
+import { Doc } from '../models/misc';
+import { Answer, FEEDBACK, RetrievalPayload } from './conversationModels';
 
 export function handleFetchAnswer(
   question: string,
@@ -54,23 +28,22 @@ export function handleFetchAnswer(
       title: any;
     }
 > {
-  const docPath = getDocPath(selectedDocs);
   history = history.map((item) => {
     return { prompt: item.prompt, response: item.response };
   });
+  const payload: RetrievalPayload = {
+    question: question,
+    history: JSON.stringify(history),
+    conversation_id: conversationId,
+    prompt_id: promptId,
+    chunks: chunks,
+    token_limit: token_limit,
+  };
+  if (selectedDocs && 'id' in selectedDocs)
+    payload.active_docs = selectedDocs.id as string;
+  payload.retriever = selectedDocs?.retriever as string;
   return conversationService
-    .answer(
-      {
-        question: question,
-        history: history,
-        active_docs: docPath,
-        conversation_id: conversationId,
-        prompt_id: promptId,
-        chunks: chunks,
-        token_limit: token_limit,
-      },
-      signal,
-    )
+    .answer(payload, signal)
     .then((response) => {
       if (response.ok) {
         return response.json();
@@ -101,16 +74,27 @@ export function handleFetchAnswerSteaming(
   token_limit: number,
   onEvent: (event: MessageEvent) => void,
 ): Promise<Answer> {
-  const docPath = getDocPath(selectedDocs);
   history = history.map((item) => {
     return { prompt: item.prompt, response: item.response };
   });
+  const payload: RetrievalPayload = {
+    question: question,
+    history: JSON.stringify(history),
+    conversation_id: conversationId,
+    prompt_id: promptId,
+    chunks: chunks,
+    token_limit: token_limit,
+  };
+  if (selectedDocs && 'id' in selectedDocs)
+    payload.active_docs = selectedDocs.id as string;
+  payload.retriever = selectedDocs?.retriever as string;
+
   return new Promise<Answer>((resolve, reject) => {
     conversationService
       .answerStream(
         {
           question: question,
-          active_docs: docPath,
+          active_docs: selectedDocs?.id as string,
           history: JSON.stringify(history),
           conversation_id: conversationId,
           prompt_id: promptId,
@@ -176,11 +160,23 @@ export function handleSearch(
   chunks: string,
   token_limit: number,
 ) {
-  const docPath = getDocPath(selectedDocs);
+  history = history.map((item) => {
+    return { prompt: item.prompt, response: item.response };
+  });
+  const payload: RetrievalPayload = {
+    question: question,
+    history: JSON.stringify(history),
+    conversation_id: conversation_id,
+    chunks: chunks,
+    token_limit: token_limit,
+  };
+  if (selectedDocs && 'id' in selectedDocs)
+    payload.active_docs = selectedDocs.id as string;
+  payload.retriever = selectedDocs?.retriever as string;
   return conversationService
     .search({
       question: question,
-      active_docs: docPath,
+      active_docs: selectedDocs?.id as string,
       conversation_id,
       history,
       chunks: chunks,
@@ -194,6 +190,27 @@ export function handleSearch(
     .catch((err) => console.log(err));
 }
 
+export function handleSearchViaApiKey(
+  question: string,
+  api_key: string,
+  history: Array<any> = [],
+) {
+  history = history.map((item) => {
+    return { prompt: item.prompt, response: item.response };
+  });
+  return conversationService
+    .search({
+      question: question,
+      history: JSON.stringify(history),
+      api_key: api_key,
+    })
+    .then((response) => response.json())
+    .then((data) => {
+      return data;
+    })
+    .catch((err) => console.log(err));
+}
+
 export function handleSendFeedback(
   prompt: string,
   response: string,

frontend/src/conversation/conversationModels.ts

@@ -31,3 +31,13 @@ export interface Query {
   conversationId?: string | null;
   title?: string | null;
 }
+export interface RetrievalPayload {
+  question: string;
+  active_docs?: string;
+  retriever?: string;
+  history: string;
+  conversation_id: string | null;
+  prompt_id?: string | null;
+  chunks: string;
+  token_limit: number;
+}

frontend/src/conversation/sharedConversationSlice.ts

@@ -6,6 +6,7 @@ import { createAsyncThunk } from '@reduxjs/toolkit';
 import {
   handleFetchSharedAnswer,
   handleFetchSharedAnswerStreaming,
+  handleSearchViaApiKey,
 } from './conversationHandlers';
 
 const API_STREAMING = import.meta.env.VITE_API_STREAMING === 'true';
@@ -44,6 +45,22 @@ export const fetchSharedAnswer = createAsyncThunk<Answer, { question: string }>(
               // set status to 'idle'
               dispatch(sharedConversationSlice.actions.setStatus('idle'));
               dispatch(saveToLocalStorage());
+
+              state.sharedConversation.apiKey &&
+                handleSearchViaApiKey(
+                  question,
+                  state.sharedConversation.apiKey,
+                  state.sharedConversation.queries,
+                ).then((sources) => {
+                  //dispatch streaming sources
+                  sources &&
+                    dispatch(
+                      updateStreamingSource({
+                        index: state.sharedConversation.queries.length - 1,
+                        query: { sources: sources ?? [] },
+                      }),
+                    );
+                });
             } else if (data.type === 'error') {
               // set status to 'failed'
               dispatch(sharedConversationSlice.actions.setStatus('failed'));
@@ -164,6 +181,20 @@ export const sharedConversationSlice = createSlice({
         ...query,
       };
     },
+    updateStreamingSource(
+      state,
+      action: PayloadAction<{ index: number; query: Partial<Query> }>,
+    ) {
+      const { index, query } = action.payload;
+      if (!state.queries[index].sources) {
+        state.queries[index].sources = query.sources ?? [];
+      } else if (query.sources && query.sources.length > 0) {
+        state.queries[index].sources = [
+          ...(state.queries[index].sources ?? []),
+          ...query.sources,
+        ];
+      }
+    },
     raiseError(
       state,
       action: PayloadAction<{ index: number; message: string }>,
@@ -213,6 +244,7 @@ export const {
   updateStreamingQuery,
   addQuery,
   saveToLocalStorage,
+  updateStreamingSource,
 } = sharedConversationSlice.actions;
 
 export const selectStatus = (state: RootState) => state.conversation.status;

frontend/src/hooks/useDefaultDocument.ts

@@ -1,7 +1,8 @@
 import React from 'react';
 import { useDispatch, useSelector } from 'react-redux';
 
-import { Doc, getDocs } from '../preferences/preferenceApi';
+import { getDocs } from '../preferences/preferenceApi';
+import { Doc } from '../models/misc';
 import {
   selectSelectedDocs,
   setSelectedDocs,

frontend/src/index.css

@@ -420,10 +420,36 @@ template {
   src: url('/fonts/Inter-Variable.ttf');
 }
 
+@font-face {
+  font-family: 'IBMPlexMono-Medium';
+  font-weight: 500;
+  src: url('/fonts/IBMPlexMono-Medium.ttf');
+}
+
 ::-webkit-scrollbar {
   width: 0;
 }
 
+input:-webkit-autofill {
+  -webkit-box-shadow: 0 0 0 50px white inset;
+}
+
+input:-webkit-autofill:focus {
+  -webkit-box-shadow: 0 0 0 50px white inset;
+}
+
+@media (prefers-color-scheme: dark) {
+  input:-webkit-autofill {
+    -webkit-box-shadow: 0 0 0 50px rgb(68, 70, 84) inset;
+    -webkit-text-fill-color: white;
+  }
+
+  input:-webkit-autofill:focus {
+    -webkit-box-shadow: 0 0 0 50px rgb(68, 70, 84) inset;
+    -webkit-text-fill-color: white;
+  }
+}
+
 .inputbox-style {
   resize: none;
   padding-left: 36px;
@@ -441,3 +467,7 @@ template {
   -webkit-box-orient: vertical;
   text-overflow: ellipsis;
 }
+
+.logs-table {
+  font-family: 'IBMPlexMono-Medium', system-ui;
+}

frontend/src/locale/en.json

@@ -62,6 +62,12 @@
       "key": "API Key",
       "sourceDoc": "Source Document",
       "createNew": "Create New"
+    },
+    "analytics": {
+      "label": "Analytics"
+    },
+    "logs": {
+      "label": "Logs"
     }
   },
   "modals": {
@@ -71,7 +77,7 @@
       "remote": "Remote",
       "name": "Name",
       "choose": "Choose Files",
-      "info": "Please upload .pdf, .txt, .rst, .docx, .md, .zip limited to 25mb",
+      "info": "Please upload .pdf, .txt, .rst, .csv, .docx, .md, .zip limited to 25mb",
       "uploadedFiles": "Uploaded Files",
       "cancel": "Cancel",
       "train": "Train",

frontend/src/locale/es.json

@@ -62,6 +62,12 @@
       "key": "Clave de API",
       "sourceDoc": "Documento Fuente",
       "createNew": "Crear Nuevo"
+    },
+    "analytics": {
+      "label": "Analítica"
+    },
+    "logs": {
+      "label": "Registros"
     }
   },
   "modals": {

frontend/src/locale/jp.json

@@ -62,6 +62,12 @@
       "key": "APIキー",
       "sourceDoc": "ソースドキュメント",
       "createNew": "新規作成"
+    },
+    "analytics": {
+      "label": "分析"
+    },
+    "logs": {
+      "label": "ログ"
     }
   },
   "modals": {

frontend/src/locale/zh.json

@@ -62,6 +62,12 @@
       "key": "API 密钥",
       "sourceDoc": "源文档",
       "createNew": "创建新的"
+    },
+    "analytics": {
+      "label": "分析"
+    },
+    "logs": {
+      "label": "日志"
     }
   },
   "modals": {

frontend/src/modals/CreateAPIKeyModal.tsx

@@ -22,8 +22,9 @@ export default function CreateAPIKeyModal({
 
   const [APIKeyName, setAPIKeyName] = React.useState<string>('');
   const [sourcePath, setSourcePath] = React.useState<{
-    label: string;
-    value: string;
+    name: string;
+    id: string;
+    type: string;
   } | null>(null);
   const [prompt, setPrompt] = React.useState<{
     name: string;
@@ -41,27 +42,17 @@ export default function CreateAPIKeyModal({
       ? docs
           .filter((doc) => doc.model === embeddingsName)
           .map((doc: Doc) => {
-            let namePath = doc.name;
-            if (doc.language === namePath) {
-              namePath = '.project';
-            }
-            let docPath = 'default';
-            if (doc.location === 'local') {
-              docPath = 'local' + '/' + doc.name + '/';
-            } else if (doc.location === 'remote') {
-              docPath =
-                doc.language +
-                '/' +
-                namePath +
-                '/' +
-                doc.version +
-                '/' +
-                doc.model +
-                '/';
+            if ('id' in doc) {
+              return {
+                name: doc.name,
+                id: doc.id as string,
+                type: 'local',
+              };
             }
             return {
-              label: doc.name,
-              value: docPath,
+              name: doc.name,
+              id: doc.id ?? 'default',
+              type: doc.type ?? 'default',
             };
           })
       : [];
@@ -107,9 +98,14 @@ export default function CreateAPIKeyModal({
           <Dropdown
             placeholder={t('modals.createAPIKey.sourceDoc')}
             selectedValue={sourcePath}
-            onSelect={(selection: { label: string; value: string }) =>
-              setSourcePath(selection)
-            }
+            onSelect={(selection: {
+              name: string;
+              id: string;
+              type: string;
+            }) => {
+              setSourcePath(selection);
+              console.log(selection);
+            }}
             options={extractDocPaths()}
             size="w-full"
             rounded="xl"
@@ -142,16 +138,22 @@ export default function CreateAPIKeyModal({
         </div>
         <button
           disabled={!sourcePath || APIKeyName.length === 0 || !prompt}
-          onClick={() =>
-            sourcePath &&
-            prompt &&
-            createAPIKey({
-              name: APIKeyName,
-              source: sourcePath.value,
-              prompt_id: prompt.id,
-              chunks: chunk,
-            })
-          }
+          onClick={() => {
+            if (sourcePath && prompt) {
+              const payload: any = {
+                name: APIKeyName,
+                prompt_id: prompt.id,
+                chunks: chunk,
+              };
+              if (sourcePath.type === 'default') {
+                payload.retriever = sourcePath.id;
+              }
+              if (sourcePath.type === 'local') {
+                payload.source = sourcePath.id;
+              }
+              createAPIKey(payload);
+            }
+          }}
           className="float-right mt-4 rounded-full bg-purple-30 px-5 py-2 text-sm text-white hover:bg-[#6F3FD1] disabled:opacity-50"
         >
           {t('modals.createAPIKey.create')}

frontend/src/modals/DeleteConvModal.tsx

@@ -19,15 +19,11 @@ export default function DeleteConvModal({
   const dispatch = useDispatch();
   const { isMobile } = useMediaQuery();
   const { t } = useTranslation();
-  useOutsideAlerter(
-    modalRef,
-    () => {
-      if (isMobile && modalState === 'ACTIVE') {
-        dispatch(setModalState('INACTIVE'));
-      }
-    },
-    [modalState],
-  );
+  useOutsideAlerter(modalRef, () => {
+    if (isMobile && modalState === 'ACTIVE') {
+      dispatch(setModalState('INACTIVE'));
+    }
+  }, [modalState]);
 
   function handleSubmit() {
     handleDeleteAllConv();

frontend/src/modals/ShareConversationModal.tsx

@@ -46,27 +46,9 @@ export const ShareConversationModal = ({
       ? docs
           .filter((doc) => doc.model === embeddingsName)
           .map((doc: Doc) => {
-            let namePath = doc.name;
-            if (doc.language === namePath) {
-              namePath = '.project';
-            }
-            let docPath = 'default';
-            if (doc.location === 'local') {
-              docPath = 'local' + '/' + doc.name + '/';
-            } else if (doc.location === 'remote') {
-              docPath =
-                doc.language +
-                '/' +
-                namePath +
-                '/' +
-                doc.version +
-                '/' +
-                doc.model +
-                '/';
-            }
             return {
               label: doc.name,
-              value: docPath,
+              value: doc.id ?? 'default',
             };
           })
       : [];

frontend/src/models/misc.ts

@@ -4,16 +4,13 @@ export type User = {
   avatar: string;
 };
 export type Doc = {
-  location: string;
+  id?: string;
   name: string;
-  language: string;
-  version: string;
-  description: string;
-  fullName: string;
   date: string;
-  docLink: string;
   model: string;
   tokens?: string;
+  type?: string;
+  retriever?: string;
 };
 
 export type PromptProps = {

frontend/src/preferences/APIKeyModal.tsx

@@ -22,15 +22,11 @@ export default function APIKeyModal({
   const modalRef = useRef(null);
   const { isMobile } = useMediaQuery();
 
-  useOutsideAlerter(
-    modalRef,
-    () => {
-      if (isMobile && modalState === 'ACTIVE') {
-        setModalState('INACTIVE');
-      }
-    },
-    [modalState],
-  );
+  useOutsideAlerter(modalRef, () => {
+    if (isMobile && modalState === 'ACTIVE') {
+      setModalState('INACTIVE');
+    }
+  }, [modalState]);
 
   function handleSubmit() {
     if (key.length <= 1) {

frontend/src/preferences/preferenceApi.ts

@@ -1,18 +1,6 @@
 import conversationService from '../api/services/conversationService';
 import userService from '../api/services/userService';
-
-// not all properties in Doc are going to be present. Make some optional
-export type Doc = {
-  location: string;
-  name: string;
-  language: string;
-  version: string;
-  description: string;
-  fullName: string;
-  date: string;
-  docLink: string;
-  model: string;
-};
+import { Doc } from '../models/misc';
 
 //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
 export async function getDocs(): Promise<Doc[] | null> {
@@ -78,17 +66,10 @@ export function setLocalPrompt(prompt: string): void {
 
 export function setLocalRecentDocs(doc: Doc | null): void {
   localStorage.setItem('DocsGPTRecentDocs', JSON.stringify(doc));
-  let namePath = doc?.name;
-  if (doc?.language === namePath) {
-    namePath = '.project';
-  }
 
   let docPath = 'default';
-  if (doc?.location === 'local') {
+  if (doc?.type === 'local') {
     docPath = 'local' + '/' + doc.name + '/';
-  } else if (doc?.location === 'remote') {
-    docPath =
-      doc.language + '/' + namePath + '/' + doc.version + '/' + doc.model + '/';
   }
   userService
     .checkDocs({

frontend/src/preferences/preferenceSlice.ts

@@ -4,9 +4,9 @@ import {
   createSlice,
   isAnyOf,
 } from '@reduxjs/toolkit';
-import { Doc, setLocalApiKey, setLocalRecentDocs } from './preferenceApi';
+import { setLocalApiKey, setLocalRecentDocs } from './preferenceApi';
 import { RootState } from '../store';
-import { ActiveState } from '../models/misc';
+import { ActiveState, Doc } from '../models/misc';
 
 interface Preference {
   apiKey: string;
@@ -25,15 +25,13 @@ const initialState: Preference = {
   chunks: '2',
   token_limit: 2000,
   selectedDocs: {
+    id: 'default',
     name: 'default',
-    language: 'default',
-    location: 'default',
-    version: 'default',
-    description: 'default',
-    fullName: 'default',
+    type: 'remote',
     date: 'default',
     docLink: 'default',
     model: 'openai_text-embedding-ada-002',
+    retriever: 'classic',
   } as Doc,
   sourceDocs: null,
   conversations: null,

frontend/src/settings/APIKeys.tsx

@@ -5,15 +5,14 @@ import userService from '../api/services/userService';
 import Trash from '../assets/trash.svg';
 import CreateAPIKeyModal from '../modals/CreateAPIKeyModal';
 import SaveAPIKeyModal from '../modals/SaveAPIKeyModal';
+import { APIKeyData } from './types';
 
 export default function APIKeys() {
   const { t } = useTranslation();
   const [isCreateModalOpen, setCreateModal] = React.useState(false);
   const [isSaveKeyModalOpen, setSaveKeyModal] = React.useState(false);
   const [newKey, setNewKey] = React.useState('');
-  const [apiKeys, setApiKeys] = React.useState<
-    { name: string; key: string; source: string; id: string }[]
-  >([]);
+  const [apiKeys, setApiKeys] = React.useState<APIKeyData[]>([]);
 
   const handleFetchKeys = async () => {
     try {
@@ -48,7 +47,8 @@ export default function APIKeys() {
 
   const handleCreateKey = (payload: {
     name: string;
-    source: string;
+    source?: string;
+    retriever?: string;
     prompt_id: string;
     chunks: string;
   }) => {
@@ -77,7 +77,7 @@ export default function APIKeys() {
   }, []);
   return (
     <div className="mt-8">
-      <div className="flex w-full flex-col lg:w-max">
+      <div className="flex flex-col max-w-[876px]">
         <div className="flex justify-end">
           <button
             onClick={() => setCreateModal(true)}
@@ -103,7 +103,7 @@ export default function APIKeys() {
             <table className="block w-max table-auto content-center justify-center rounded-xl border text-center dark:border-chinese-silver dark:text-bright-gray">
               <thead>
                 <tr>
-                  <th className="border-r p-4 md:w-[244px]">
+                  <th className="w-[244px] border-r p-4">
                     {t('settings.apiKeys.name')}
                   </th>
                   <th className="w-[244px] border-r px-4 py-2">

frontend/src/settings/Analytics.tsx

@@ -0,0 +1,390 @@
+import {
+  BarElement,
+  CategoryScale,
+  Chart as ChartJS,
+  Legend,
+  LinearScale,
+  Title,
+  Tooltip,
+} from 'chart.js';
+import React from 'react';
+import { Bar } from 'react-chartjs-2';
+
+import userService from '../api/services/userService';
+import Dropdown from '../components/Dropdown';
+import { htmlLegendPlugin } from '../utils/chartUtils';
+import { formatDate } from '../utils/dateTimeUtils';
+import { APIKeyData } from './types';
+
+import type { ChartData } from 'chart.js';
+ChartJS.register(
+  CategoryScale,
+  LinearScale,
+  BarElement,
+  Title,
+  Tooltip,
+  Legend,
+);
+
+const filterOptions = [
+  { label: 'Hour', value: 'last_hour' },
+  { label: '24 Hours', value: 'last_24_hour' },
+  { label: '7 Days', value: 'last_7_days' },
+  { label: '15 Days', value: 'last_15_days' },
+  { label: '30 Days', value: 'last_30_days' },
+];
+
+export default function Analytics() {
+  const [messagesData, setMessagesData] = React.useState<Record<
+    string,
+    number
+  > | null>(null);
+  const [tokenUsageData, setTokenUsageData] = React.useState<Record<
+    string,
+    number
+  > | null>(null);
+  const [feedbackData, setFeedbackData] = React.useState<Record<
+    string,
+    {
+      positive: number;
+      negative: number;
+    }
+  > | null>(null);
+  const [chatbots, setChatbots] = React.useState<APIKeyData[]>([]);
+  const [selectedChatbot, setSelectedChatbot] =
+    React.useState<APIKeyData | null>();
+  const [messagesFilter, setMessagesFilter] = React.useState<{
+    label: string;
+    value: string;
+  }>({ label: '30 Days', value: 'last_30_days' });
+  const [tokenUsageFilter, setTokenUsageFilter] = React.useState<{
+    label: string;
+    value: string;
+  }>({ label: '30 Days', value: 'last_30_days' });
+  const [feedbackFilter, setFeedbackFilter] = React.useState<{
+    label: string;
+    value: string;
+  }>({ label: '30 Days', value: 'last_30_days' });
+
+  const fetchChatbots = async () => {
+    try {
+      const response = await userService.getAPIKeys();
+      if (!response.ok) {
+        throw new Error('Failed to fetch Chatbots');
+      }
+      const chatbots = await response.json();
+      setChatbots(chatbots);
+    } catch (error) {
+      console.error(error);
+    }
+  };
+
+  const fetchMessagesData = async (chatbot_id?: string, filter?: string) => {
+    try {
+      const response = await userService.getMessageAnalytics({
+        api_key_id: chatbot_id,
+        filter_option: filter,
+      });
+      if (!response.ok) {
+        throw new Error('Failed to fetch analytics data');
+      }
+      const data = await response.json();
+      setMessagesData(data.messages);
+    } catch (error) {
+      console.error(error);
+    }
+  };
+
+  const fetchTokenData = async (chatbot_id?: string, filter?: string) => {
+    try {
+      const response = await userService.getTokenAnalytics({
+        api_key_id: chatbot_id,
+        filter_option: filter,
+      });
+      if (!response.ok) {
+        throw new Error('Failed to fetch analytics data');
+      }
+      const data = await response.json();
+      setTokenUsageData(data.token_usage);
+    } catch (error) {
+      console.error(error);
+    }
+  };
+
+  const fetchFeedbackData = async (chatbot_id?: string, filter?: string) => {
+    try {
+      const response = await userService.getFeedbackAnalytics({
+        api_key_id: chatbot_id,
+        filter_option: filter,
+      });
+      if (!response.ok) {
+        throw new Error('Failed to fetch analytics data');
+      }
+      const data = await response.json();
+      setFeedbackData(data.feedback);
+    } catch (error) {
+      console.error(error);
+    }
+  };
+
+  React.useEffect(() => {
+    fetchChatbots();
+  }, []);
+
+  React.useEffect(() => {
+    const id = selectedChatbot?.id;
+    const filter = messagesFilter;
+    fetchMessagesData(id, filter?.value);
+  }, [selectedChatbot, messagesFilter]);
+
+  React.useEffect(() => {
+    const id = selectedChatbot?.id;
+    const filter = tokenUsageFilter;
+    fetchTokenData(id, filter?.value);
+  }, [selectedChatbot, tokenUsageFilter]);
+
+  React.useEffect(() => {
+    const id = selectedChatbot?.id;
+    const filter = feedbackFilter;
+    fetchFeedbackData(id, filter?.value);
+  }, [selectedChatbot, feedbackFilter]);
+  return (
+    <div className="mt-12">
+      <div className="flex flex-col items-start">
+        <div className="flex flex-col gap-3">
+          <p className="font-bold text-jet dark:text-bright-gray">
+            Filter by chatbot
+          </p>
+          <Dropdown
+            size="w-[55vw] sm:w-[360px]"
+            options={[
+              ...chatbots.map((chatbot) => ({
+                label: chatbot.name,
+                value: chatbot.id,
+              })),
+              { label: 'None', value: '' },
+            ]}
+            placeholder="Select chatbot"
+            onSelect={(chatbot: { label: string; value: string }) => {
+              setSelectedChatbot(
+                chatbots.find((item) => item.id === chatbot.value),
+              );
+            }}
+            selectedValue={
+              (selectedChatbot && {
+                label: selectedChatbot.name,
+                value: selectedChatbot.id,
+              }) ||
+              null
+            }
+            rounded="3xl"
+            border="border"
+          />
+        </div>
+        <div className="mt-8 w-full flex flex-col sm:flex-row gap-3">
+          <div className="h-[345px] sm:w-1/2 w-full px-6 py-5 border rounded-2xl border-silver dark:border-silver/40">
+            <div className="flex flex-row items-center justify-start gap-3">
+              <p className="font-bold text-jet dark:text-bright-gray">
+                Messages
+              </p>
+              <Dropdown
+                size="w-[125px]"
+                options={filterOptions}
+                placeholder="Filter"
+                onSelect={(selectedOption: {
+                  label: string;
+                  value: string;
+                }) => {
+                  setMessagesFilter(selectedOption);
+                }}
+                selectedValue={messagesFilter ?? null}
+                rounded="3xl"
+                border="border"
+                contentSize="text-sm"
+              />
+            </div>
+            <div className="mt-px relative h-[245px] w-full">
+              <div
+                id="legend-container-1"
+                className="flex flex-row items-center justify-end"
+              ></div>
+              <AnalyticsChart
+                data={{
+                  labels: Object.keys(messagesData || {}).map((item) =>
+                    formatDate(item),
+                  ),
+                  datasets: [
+                    {
+                      label: 'Messages',
+                      data: Object.values(messagesData || {}),
+                      backgroundColor: '#7D54D1',
+                    },
+                  ],
+                }}
+                legendID="legend-container-1"
+                maxTicksLimitInX={8}
+                isStacked={false}
+              />
+            </div>
+          </div>
+          <div className="h-[345px] sm:w-1/2 w-full px-6 py-5 border rounded-2xl border-silver dark:border-silver/40">
+            <div className="flex flex-row items-center justify-start gap-3">
+              <p className="font-bold text-jet dark:text-bright-gray">
+                Token Usage
+              </p>
+              <Dropdown
+                size="w-[125px]"
+                options={filterOptions}
+                placeholder="Filter"
+                onSelect={(selectedOption: {
+                  label: string;
+                  value: string;
+                }) => {
+                  setTokenUsageFilter(selectedOption);
+                }}
+                selectedValue={tokenUsageFilter ?? null}
+                rounded="3xl"
+                border="border"
+                contentSize="text-sm"
+              />
+            </div>
+            <div className="mt-px relative h-[245px] w-full">
+              <div
+                id="legend-container-2"
+                className="flex flex-row items-center justify-end"
+              ></div>
+              <AnalyticsChart
+                data={{
+                  labels: Object.keys(tokenUsageData || {}).map((item) =>
+                    formatDate(item),
+                  ),
+                  datasets: [
+                    {
+                      label: 'Tokens',
+                      data: Object.values(tokenUsageData || {}),
+                      backgroundColor: '#7D54D1',
+                    },
+                  ],
+                }}
+                legendID="legend-container-2"
+                maxTicksLimitInX={8}
+                isStacked={false}
+              />
+            </div>
+          </div>
+        </div>
+        <div className="mt-8 w-full">
+          <div className="h-[345px] w-full px-6 py-5 border rounded-2xl border-silver dark:border-silver/40">
+            <div className="flex flex-row items-center justify-start gap-3">
+              <p className="font-bold text-jet dark:text-bright-gray">
+                User Feedback
+              </p>
+              <Dropdown
+                size="w-[125px]"
+                options={filterOptions}
+                placeholder="Filter"
+                onSelect={(selectedOption: {
+                  label: string;
+                  value: string;
+                }) => {
+                  setFeedbackFilter(selectedOption);
+                }}
+                selectedValue={feedbackFilter ?? null}
+                rounded="3xl"
+                border="border"
+                contentSize="text-sm"
+              />
+            </div>
+            <div className="mt-px relative h-[245px] w-full">
+              <div
+                id="legend-container-3"
+                className="flex flex-row items-center justify-end"
+              ></div>
+              <AnalyticsChart
+                data={{
+                  labels: Object.keys(feedbackData || {}).map((item) =>
+                    formatDate(item),
+                  ),
+                  datasets: [
+                    {
+                      label: 'Positive',
+                      data: Object.values(feedbackData || {}).map(
+                        (item) => item.positive,
+                      ),
+                      backgroundColor: '#8BD154',
+                    },
+                    {
+                      label: 'Negative',
+                      data: Object.values(feedbackData || {}).map(
+                        (item) => item.negative,
+                      ),
+                      backgroundColor: '#D15454',
+                    },
+                  ],
+                }}
+                legendID="legend-container-3"
+                maxTicksLimitInX={10}
+                isStacked={true}
+              />
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}
+
+type AnalyticsChartProps = {
+  data: ChartData<'bar'>;
+  legendID: string;
+  maxTicksLimitInX: number;
+  isStacked: boolean;
+};
+
+function AnalyticsChart({
+  data,
+  legendID,
+  maxTicksLimitInX,
+  isStacked,
+}: AnalyticsChartProps) {
+  const options = {
+    responsive: true,
+    maintainAspectRatio: false,
+    plugins: {
+      legend: {
+        display: false,
+      },
+      htmlLegend: {
+        containerID: legendID,
+      },
+    },
+    scales: {
+      x: {
+        grid: {
+          lineWidth: 0.2,
+          color: '#C4C4C4',
+        },
+        border: {
+          width: 0.2,
+          color: '#C4C4C4',
+        },
+        ticks: {
+          maxTicksLimit: maxTicksLimitInX,
+        },
+        stacked: isStacked,
+      },
+      y: {
+        grid: {
+          lineWidth: 0.2,
+          color: '#C4C4C4',
+        },
+        border: {
+          width: 0.2,
+          color: '#C4C4C4',
+        },
+        stacked: isStacked,
+      },
+    },
+  };
+  return <Bar options={options} plugins={[htmlLegendPlugin]} data={data} />;
+}

frontend/src/settings/Documents.tsx

@@ -61,12 +61,10 @@ const Documents: React.FC<DocumentsProps> = ({
                       {document.tokens ? formatTokens(+document.tokens) : ''}
                     </td>
                     <td className="border-r border-t px-4 py-2">
-                      {document.location === 'remote'
-                        ? 'Pre-loaded'
-                        : 'Private'}
+                      {document.type === 'remote' ? 'Pre-loaded' : 'Private'}
                     </td>
                     <td className="border-t px-4 py-2">
-                      {document.location !== 'remote' && (
+                      {document.type !== 'remote' && (
                         <img
                           src={Trash}
                           alt="Delete"

frontend/src/settings/General.tsx

@@ -89,7 +89,7 @@ export default function General() {
     changeLanguage(selectedLanguage?.value);
   }, [selectedLanguage, changeLanguage]);
   return (
-    <div className="mt-[59px]">
+    <div className="mt-12">
       <div className="mb-5">
         <p className="font-bold text-jet dark:text-bright-gray">
           {t('settings.general.selectTheme')}

frontend/src/settings/Logs.tsx

@@ -0,0 +1,175 @@
+import React from 'react';
+
+import userService from '../api/services/userService';
+import ChevronRight from '../assets/chevron-right.svg';
+import Dropdown from '../components/Dropdown';
+import { APIKeyData, LogData } from './types';
+import CoppyButton from '../components/CopyButton';
+
+export default function Logs() {
+  const [chatbots, setChatbots] = React.useState<APIKeyData[]>([]);
+  const [selectedChatbot, setSelectedChatbot] =
+    React.useState<APIKeyData | null>();
+  const [logs, setLogs] = React.useState<LogData[]>([]);
+  const [page, setPage] = React.useState(1);
+  const [hasMore, setHasMore] = React.useState(true);
+
+  const fetchChatbots = async () => {
+    try {
+      const response = await userService.getAPIKeys();
+      if (!response.ok) {
+        throw new Error('Failed to fetch Chatbots');
+      }
+      const chatbots = await response.json();
+      setChatbots(chatbots);
+    } catch (error) {
+      console.error(error);
+    }
+  };
+
+  const fetchLogs = async () => {
+    try {
+      const response = await userService.getLogs({
+        page: page,
+        api_key_id: selectedChatbot?.id,
+        page_size: 10,
+      });
+      if (!response.ok) {
+        throw new Error('Failed to fetch logs');
+      }
+      const olderLogs = await response.json();
+      setLogs([...logs, ...olderLogs.logs]);
+      setHasMore(olderLogs.has_more);
+    } catch (error) {
+      console.error(error);
+    }
+  };
+
+  React.useEffect(() => {
+    fetchChatbots();
+  }, []);
+
+  React.useEffect(() => {
+    if (hasMore) fetchLogs();
+  }, [page, selectedChatbot]);
+  return (
+    <div className="mt-12">
+      <div className="flex flex-col items-start">
+        <div className="flex flex-col gap-3">
+          <p className="font-bold text-jet dark:text-bright-gray">
+            Filter by chatbot
+          </p>
+          <Dropdown
+            size="w-[55vw] sm:w-[360px]"
+            options={[
+              ...chatbots.map((chatbot) => ({
+                label: chatbot.name,
+                value: chatbot.id,
+              })),
+              { label: 'None', value: '' },
+            ]}
+            placeholder="Select chatbot"
+            onSelect={(chatbot: { label: string; value: string }) => {
+              setSelectedChatbot(
+                chatbots.find((item) => item.id === chatbot.value),
+              );
+              setLogs([]);
+              setPage(1);
+              setHasMore(true);
+            }}
+            selectedValue={
+              (selectedChatbot && {
+                label: selectedChatbot.name,
+                value: selectedChatbot.id,
+              }) ||
+              null
+            }
+            rounded="3xl"
+            border="border"
+          />
+        </div>
+      </div>
+      <div className="mt-8">
+        <LogsTable logs={logs} setPage={setPage} />
+      </div>
+    </div>
+  );
+}
+
+type LogsTableProps = {
+  logs: LogData[];
+  setPage: React.Dispatch<React.SetStateAction<number>>;
+};
+
+function LogsTable({ logs, setPage }: LogsTableProps) {
+  const observerRef = React.useRef<any>();
+  const firstObserver = React.useCallback((node: HTMLDivElement) => {
+    if (observerRef.current) {
+      observerRef.current = new IntersectionObserver((enteries) => {
+        if (enteries[0].isIntersecting) setPage((prev) => prev + 1);
+      });
+    }
+    if (node && observerRef.current) observerRef.current.observe(node);
+  }, []);
+  return (
+    <div className="logs-table border rounded-2xl h-[55vh] w-full overflow-hidden border-silver dark:border-silver/40">
+      <div className="h-8 bg-black/10 dark:bg-chinese-black flex flex-col items-start justify-center">
+        <p className="px-3 text-xs dark:text-gray-6000">
+          API generated / chatbot conversations
+        </p>
+      </div>
+      <div
+        ref={observerRef}
+        className="flex flex-col items-start h-[51vh] overflow-y-auto bg-transparent flex-grow gap-px"
+      >
+        {logs.map((log, index) => {
+          if (index === logs.length - 1) {
+            return (
+              <div ref={firstObserver} key={index}>
+                <Log log={log} />
+              </div>
+            );
+          } else return <Log key={index} log={log} />;
+        })}
+      </div>
+    </div>
+  );
+}
+
+function Log({ log }: { log: LogData }) {
+  const logLevelColor = {
+    info: 'text-green-500',
+    error: 'text-red-500',
+    warning: 'text-yellow-500',
+  };
+  const { id, action, timestamp, ...filteredLog } = log;
+  return (
+    <details className="group bg-transparent [&_summary::-webkit-details-marker]:hidden w-full hover:bg-[#F9F9F9] hover:dark:bg-dark-charcoal">
+      <summary className="flex flex-row items-center gap-2 text-gray-900 cursor-pointer p-2 group-open:bg-[#F9F9F9] dark:group-open:bg-dark-charcoal">
+        <img
+          src={ChevronRight}
+          alt="chevron-right"
+          className="w-3 h-3 transition duration-300 group-open:rotate-90"
+        />
+        <span className="flex flex-row gap-2">
+          <h2 className="text-xs text-black/60 dark:text-bright-gray">{`${log.timestamp}`}</h2>
+          <h2 className="text-xs text-[#913400] dark:text-[#DF5200]">{`[${log.action}]`}</h2>
+          <h2
+            className={`text-xs ${logLevelColor[log.level]}`}
+          >{`${log.question}`}</h2>
+        </span>
+      </summary>
+      <div className="px-4 group-open:bg-[#F9F9F9] dark:group-open:bg-dark-charcoal">
+        <p className="px-2 leading-relaxed text-gray-700 dark:text-gray-400 text-xs">
+          {JSON.stringify(filteredLog, null, 2)}
+        </p>
+        <div className="my-px w-8">
+          <CoppyButton
+            text={JSON.stringify(filteredLog)}
+            colorLight="transparent"
+          />
+        </div>
+      </div>
+    </details>
+  );
+}

frontend/src/settings/index.tsx

@@ -6,14 +6,16 @@ import userService from '../api/services/userService';
 import ArrowLeft from '../assets/arrow-left.svg';
 import ArrowRight from '../assets/arrow-right.svg';
 import i18n from '../locale/i18n';
-import { Doc } from '../preferences/preferenceApi';
+import { Doc } from '../models/misc';
 import {
   selectSourceDocs,
   setSourceDocs,
 } from '../preferences/preferenceSlice';
+import Analytics from './Analytics';
 import APIKeys from './APIKeys';
 import Documents from './Documents';
 import General from './General';
+import Logs from './Logs';
 import Widgets from './Widgets';
 
 export default function Settings() {
@@ -23,6 +25,8 @@ export default function Settings() {
     t('settings.general.label'),
     t('settings.documents.label'),
     t('settings.apiKeys.label'),
+    t('settings.analytics.label'),
+    t('settings.logs.label'),
   ];
   const [activeTab, setActiveTab] = React.useState(t('settings.general.label'));
   const [widgetScreenshot, setWidgetScreenshot] = React.useState<File | null>(
@@ -35,9 +39,8 @@ export default function Settings() {
   };
 
   const handleDeleteClick = (index: number, doc: Doc) => {
-    const docPath = 'indexes/' + 'local' + '/' + doc.name;
     userService
-      .deletePath(docPath)
+      .deletePath(doc.id ?? '')
       .then((response) => {
         if (response.ok && documents) {
           const updatedDocuments = [
@@ -129,6 +132,10 @@ export default function Settings() {
         );
       case t('settings.apiKeys.label'):
         return <APIKeys />;
+      case t('settings.analytics.label'):
+        return <Analytics />;
+      case t('settings.logs.label'):
+        return <Logs />;
       default:
         return null;
     }

frontend/src/settings/types/index.ts

@@ -0,0 +1,20 @@
+export type APIKeyData = {
+  id: string;
+  name: string;
+  key: string;
+  source: string;
+  prompt_id: string;
+  chunks: string;
+};
+
+export type LogData = {
+  id: string;
+  action: string;
+  level: 'info' | 'error' | 'warning';
+  user: string;
+  question: string;
+  response: string;
+  sources: Record<string, any>[];
+  retriever_params: Record<string, any>;
+  timestamp: string;
+};

frontend/src/store.ts

@@ -26,15 +26,12 @@ const store = configureStore({
       conversations: null,
       sourceDocs: [
         {
-          location: '',
-          language: '',
           name: 'default',
-          version: '',
           date: '',
-          description: '',
-          docLink: '',
-          fullName: '',
           model: '1.0',
+          type: 'remote',
+          id: 'default',
+          retriever: 'clasic',
         },
       ],
       modalState: 'INACTIVE',

frontend/src/upload/Upload.tsx

@@ -120,7 +120,7 @@ function Upload({
                     dispatch(setSourceDocs(data));
                     dispatch(
                       setSelectedDocs(
-                        data?.find((d) => d.location.toLowerCase() === 'local'),
+                        data?.find((d) => d.type?.toLowerCase() === 'local'),
                       ),
                     );
                   });
@@ -137,7 +137,7 @@ function Upload({
                     dispatch(setSourceDocs(data));
                     dispatch(
                       setSelectedDocs(
-                        data?.find((d) => d.location.toLowerCase() === 'local'),
+                        data?.find((d) => d.type?.toLowerCase() === 'local'),
                       ),
                     );
                   });
@@ -259,6 +259,7 @@ function Upload({
       'application/zip': ['.zip'],
       'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
         ['.docx'],
+      'text/csv': ['.csv'],
     },
   });
 
@@ -274,7 +275,7 @@ function Upload({
     } else
       setRedditData({
         ...redditData,
-        [name]: value,
+        [name]: name === 'number_posts' ? parseInt(value) : value,
       });
   };
 
@@ -320,6 +321,7 @@ function Upload({
               colorVariant="gray"
               value={docName}
               onChange={(e) => setDocName(e.target.value)}
+              borderVariant="thin"
             ></Input>
             <div className="relative bottom-12 left-2 mt-[-20px]">
               <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
@@ -355,6 +357,7 @@ function Upload({
         {activeTab === 'remote' && (
           <>
             <Dropdown
+              border="border"
               options={urlOptions}
               selectedValue={urlType}
               onSelect={(value: { label: string; value: string }) =>
@@ -370,6 +373,7 @@ function Upload({
                   type="text"
                   value={urlName}
                   onChange={(e) => setUrlName(e.target.value)}
+                  borderVariant="thin"
                 ></Input>
                 <div className="relative bottom-12 left-2 mt-[-20px]">
                   <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
@@ -381,6 +385,7 @@ function Upload({
                   type="text"
                   value={url}
                   onChange={(e) => setUrl(e.target.value)}
+                  borderVariant="thin"
                 ></Input>
                 <div className="relative bottom-12 left-2 mt-[-20px]">
                   <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
@@ -389,68 +394,83 @@ function Upload({
                 </div>
               </>
             ) : (
-              <>
-                <Input
-                  placeholder="Enter client ID"
-                  type="text"
-                  name="client_id"
-                  value={redditData.client_id}
-                  onChange={handleChange}
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.reddit.id')}
-                  </span>
+              <div className="flex flex-col gap-1 mt-2">
+                <div>
+                  <Input
+                    placeholder="Enter client ID"
+                    type="text"
+                    name="client_id"
+                    value={redditData.client_id}
+                    onChange={handleChange}
+                    borderVariant="thin"
+                  ></Input>
+                  <div className="relative bottom-[52px] left-2">
+                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                      {t('modals.uploadDoc.reddit.id')}
+                    </span>
+                  </div>
                 </div>
-                <Input
-                  placeholder="Enter client secret"
-                  type="text"
-                  name="client_secret"
-                  value={redditData.client_secret}
-                  onChange={handleChange}
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.reddit.secret')}
-                  </span>
+                <div>
+                  <Input
+                    placeholder="Enter client secret"
+                    type="text"
+                    name="client_secret"
+                    value={redditData.client_secret}
+                    onChange={handleChange}
+                    borderVariant="thin"
+                  ></Input>
+                  <div className="relative bottom-[52px] left-2">
+                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                      {t('modals.uploadDoc.reddit.secret')}
+                    </span>
+                  </div>
                 </div>
-                <Input
-                  placeholder="Enter user agent"
-                  type="text"
-                  name="user_agent"
-                  value={redditData.user_agent}
-                  onChange={handleChange}
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.reddit.agent')}
-                  </span>
+                <div>
+                  <Input
+                    placeholder="Enter user agent"
+                    type="text"
+                    name="user_agent"
+                    value={redditData.user_agent}
+                    onChange={handleChange}
+                    borderVariant="thin"
+                  ></Input>
+                  <div className="relative bottom-[52px] left-2">
+                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                      {t('modals.uploadDoc.reddit.agent')}
+                    </span>
+                  </div>
                 </div>
-                <Input
-                  placeholder="Enter search queries"
-                  type="text"
-                  name="search_queries"
-                  value={redditData.search_queries}
-                  onChange={handleChange}
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.reddit.searchQueries')}
-                  </span>
+                <div>
+                  <Input
+                    placeholder="Enter search queries"
+                    type="text"
+                    name="search_queries"
+                    value={redditData.search_queries}
+                    onChange={handleChange}
+                    borderVariant="thin"
+                  ></Input>
+                  <div className="relative bottom-[52px] left-2">
+                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                      {t('modals.uploadDoc.reddit.searchQueries')}
+                    </span>
+                  </div>
                 </div>
-                <Input
-                  placeholder="Enter number of posts"
-                  type="number"
-                  name="number_posts"
-                  value={redditData.number_posts}
-                  onChange={handleChange}
-                ></Input>
-                <div className="relative bottom-12 left-2 mt-[-20px]">
-                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-                    {t('modals.uploadDoc.reddit.numberOfPosts')}
-                  </span>
+                <div>
+                  <Input
+                    placeholder="Enter number of posts"
+                    type="number"
+                    name="number_posts"
+                    value={redditData.number_posts}
+                    onChange={handleChange}
+                    borderVariant="thin"
+                  ></Input>
+                  <div className="relative bottom-[52px] left-2">
+                    <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                      {t('modals.uploadDoc.reddit.numberOfPosts')}
+                    </span>
+                  </div>
                 </div>
-              </>
+              </div>
             )}
           </>
         )}

frontend/src/utils/chartUtils.ts

@@ -0,0 +1,77 @@
+import { Chart as ChartJS } from 'chart.js';
+
+const getOrCreateLegendList = (
+  chart: ChartJS,
+  id: string,
+): HTMLUListElement => {
+  const legendContainer = document.getElementById(id);
+  let listContainer = legendContainer?.querySelector('ul') as HTMLUListElement;
+
+  if (!listContainer) {
+    listContainer = document.createElement('ul');
+    listContainer.style.display = 'flex';
+    listContainer.style.flexDirection = 'row';
+    listContainer.style.margin = '0';
+    listContainer.style.padding = '0';
+
+    legendContainer?.appendChild(listContainer);
+  }
+
+  return listContainer;
+};
+
+export const htmlLegendPlugin = {
+  id: 'htmlLegend',
+  afterUpdate(chart: ChartJS, args: any, options: { containerID: string }) {
+    const ul = getOrCreateLegendList(chart, options.containerID);
+
+    while (ul.firstChild) {
+      ul.firstChild.remove();
+    }
+
+    const items =
+      chart.options.plugins?.legend?.labels?.generateLabels?.(chart) || [];
+
+    items.forEach((item: any) => {
+      const li = document.createElement('li');
+      li.style.alignItems = 'center';
+      li.style.cursor = 'pointer';
+      li.style.display = 'flex';
+      li.style.flexDirection = 'row';
+      li.style.marginLeft = '10px';
+
+      li.onclick = () => {
+        chart.setDatasetVisibility(
+          item.datasetIndex,
+          !chart.isDatasetVisible(item.datasetIndex),
+        );
+        chart.update();
+      };
+
+      const boxSpan = document.createElement('span');
+      boxSpan.style.background = item.fillStyle;
+      boxSpan.style.borderColor = item.strokeStyle;
+      boxSpan.style.borderWidth = item.lineWidth + 'px';
+      boxSpan.style.display = 'inline-block';
+      boxSpan.style.flexShrink = '0';
+      boxSpan.style.height = '10px';
+      boxSpan.style.marginRight = '10px';
+      boxSpan.style.width = '10px';
+      boxSpan.style.borderRadius = '10px';
+
+      const textContainer = document.createElement('p');
+      textContainer.style.fontSize = '12px';
+      textContainer.style.color = item.fontColor;
+      textContainer.style.margin = '0';
+      textContainer.style.padding = '0';
+      textContainer.style.textDecoration = item.hidden ? 'line-through' : '';
+
+      const text = document.createTextNode(item.text);
+      textContainer.appendChild(text);
+
+      li.appendChild(boxSpan);
+      li.appendChild(textContainer);
+      ul.appendChild(li);
+    });
+  },
+};

frontend/src/utils/dateTimeUtils.ts

@@ -0,0 +1,20 @@
+export function formatDate(dateString: string): string {
+  if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(dateString)) {
+    const dateTime = new Date(dateString);
+    return dateTime.toLocaleTimeString([], {
+      hour: '2-digit',
+      minute: '2-digit',
+    });
+  } else if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$/.test(dateString)) {
+    const dateTime = new Date(dateString);
+    return dateTime.toLocaleTimeString([], {
+      hour: '2-digit',
+      minute: '2-digit',
+    });
+  } else if (/^\d{4}-\d{2}-\d{2}$/.test(dateString)) {
+    const date = new Date(dateString);
+    return date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
+  } else {
+    return dateString;
+  }
+}

mock-backend/package.json

@@ -13,7 +13,7 @@
   "license": "ISC",
   "dependencies": {
     "cors": "^2.8.5",
-    "json-server": "^0.17.4",
+    "json-server": "^1.0.0-beta.2",
     "uuid": "^9.0.1"
   },
   "devDependencies": {

scripts/migrate_to_v1_vectorstore.py

@@ -0,0 +1,55 @@
+import pymongo
+import os
+
+def migrate_to_v1_vectorstore_mongo():
+    client = pymongo.MongoClient("mongodb://localhost:27017/")
+    db = client["docsgpt"]
+    vectors_collection = db["vectors"]
+    sources_collection = db["sources"]
+
+    for vector in vectors_collection.find():
+        if "location" in vector:
+            del vector["location"]
+        if "retriever" not in vector:
+            vector["retriever"] = "classic"
+            vector["remote_data"] = None
+        vectors_collection.update_one({"_id": vector["_id"]}, {"$set": vector})
+
+    # move data from vectors_collection to sources_collection
+    for vector in vectors_collection.find():
+        sources_collection.insert_one(vector)
+
+    vectors_collection.drop()
+
+    client.close()
+
+def migrate_faiss_to_v1_vectorstore():
+    client = pymongo.MongoClient("mongodb://localhost:27017/")
+    db = client["docsgpt"]
+    vectors_collection = db["vectors"]
+
+    for vector in vectors_collection.find():
+        old_path = f"./application/indexes/{vector['user']}/{vector['name']}"
+        new_path = f"./application/indexes/{vector['_id']}"
+        try:
+            os.rename(old_path, new_path)
+        except OSError as e:
+            print(f"Error moving {old_path} to {new_path}: {e}")
+
+    client.close()
+
+def migrate_mongo_atlas_vector_to_v1_vectorstore():
+    client = pymongo.MongoClient("mongodb+srv://<username>:<password>@<cluster>/<dbname>?retryWrites=true&w=majority")
+    db = client["docsgpt"]
+    vectors_collection = db["vectors"]
+
+    # mongodb atlas collection
+    documents_collection = db["documents"]
+
+    for vector in vectors_collection.find():
+        documents_collection.update_many({"store": vector["user"] + "/" + vector["name"]}, {"$set": {"source_id": str(vector["_id"])}})
+
+    client.close()
+
+migrate_faiss_to_v1_vectorstore()
+migrate_to_v1_vectorstore_mongo()

scripts/requirements.txt

@@ -9,7 +9,7 @@ javalang==0.13.0
 langchain==0.1.4
 langchain_community==0.2.9
 langchain-openai==0.0.5
-nltk==3.8.1
+nltk==3.9
 openapi3_parser==1.1.16
 pandas==2.2.0
 PyPDF2==3.0.1