parser functions change

token_func proposed change to chunking. open_ai_func proposed change to embedding_pipeline. Late chunking first  implementation requires further testing.

README.md

@@ -27,9 +27,7 @@ Say goodbye to time-consuming manual searches, and let <strong><a href="https://
 
 We're eager to provide personalized assistance when deploying your DocsGPT to a live environment.
 
-<a href ="https://cal.com/arc53/docsgpt-demo-b2b">
-    <img alt="Let's chat" src="https://cal.com/book-with-cal-dark.svg" />
-</a>
+[Book a Meeting :wave:](https://cal.com/arc53/docsgpt-demo-b2b)⁠
 
 [Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
 

application/api/answer/routes.py

@@ -141,17 +141,17 @@ def save_conversation(conversation_id, question, response, source_log_docs, llm)
                 "role": "assistant",
                 "content": "Summarise following conversation in no more than 3 "
                 "words, respond ONLY with the summary, use the same "
-                "language as the system \n\nUser: "
-                + question
-                + "\n\n"
-                + "AI: "
-                + response,
+                "language as the system",
             },
             {
                 "role": "user",
                 "content": "Summarise following conversation in no more than 3 words, "
                 "respond ONLY with the summary, use the same language as the "
-                "system",
+                "system \n\nUser: "
+                + question
+                + "\n\n"
+                + "AI: "
+                + response,
             },
         ]
 
@@ -241,6 +241,7 @@ def complete_stream(
         yield f"data: {data}\n\n"
     except Exception as e:
         print("\033[91merr", str(e), file=sys.stderr)
+        traceback.print_exc()
         data = json.dumps(
             {
                 "type": "error",

application/api/user/routes.py

@@ -316,7 +316,7 @@ class UploadFile(Resource):
                 for file in files:
                     filename = secure_filename(file.filename)
                     file.save(os.path.join(temp_dir, filename))
-
+                    print(f"Saved file: {filename}")
                 zip_path = shutil.make_archive(
                     base_name=os.path.join(save_dir, job_name),
                     format="zip",
@@ -324,6 +324,26 @@ class UploadFile(Resource):
                 )
                 final_filename = os.path.basename(zip_path)
                 shutil.rmtree(temp_dir)
+                task = ingest.delay(
+                    settings.UPLOAD_FOLDER,
+                    [
+                        ".rst",
+                        ".md",
+                        ".pdf",
+                        ".txt",
+                        ".docx",
+                        ".csv",
+                        ".epub",
+                        ".html",
+                        ".mdx",
+                        ".json",
+                        ".xlsx",
+                        ".pptx",
+                    ],
+                    job_name,
+                    final_filename,
+                    user,
+                )
             else:
                 file = files[0]
                 final_filename = secure_filename(file.filename)
@@ -350,9 +370,10 @@ class UploadFile(Resource):
                     final_filename,
                     user,
                 )
-        except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
+        except Exception as err:
+            print(f"Error: {err}")
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
         return make_response(jsonify({"success": True, "task_id": task.id}), 200)
 
 
@@ -423,6 +444,11 @@ class TaskStatus(Resource):
 
             task = celery.AsyncResult(task_id)
             task_meta = task.info
+            print(f"Task status: {task.status}")
+            if not isinstance(
+                task_meta, (dict, list, str, int, float, bool, type(None))
+            ):
+                task_meta = str(task_meta)  # Convert to a string representation
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 

application/llm/docsgpt_provider.py

@@ -9,35 +9,25 @@ class DocsGPTAPILLM(BaseLLM):
         super().__init__(*args, **kwargs)
         self.api_key = api_key
         self.user_api_key = user_api_key
-        self.endpoint = "https://llm.docsgpt.co.uk"
+        self.endpoint = "https://llm.arc53.com"
 
     def _raw_gen(self, baseself, model, messages, stream=False, *args, **kwargs):
-        context = messages[0]["content"]
-        user_question = messages[-1]["content"]
-        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
         response = requests.post(
-            f"{self.endpoint}/answer", json={"prompt": prompt, "max_new_tokens": 30}
+            f"{self.endpoint}/answer", json={"messages": messages, "max_new_tokens": 30}
         )
         response_clean = response.json()["a"].replace("###", "")
 
         return response_clean
 
     def _raw_gen_stream(self, baseself, model, messages, stream=True, *args, **kwargs):
-        context = messages[0]["content"]
-        user_question = messages[-1]["content"]
-        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
-        # send prompt to endpoint /stream
         response = requests.post(
             f"{self.endpoint}/stream",
-            json={"prompt": prompt, "max_new_tokens": 256},
+            json={"messages": messages, "max_new_tokens": 256},
             stream=True,
         )
 
         for line in response.iter_lines():
             if line:
-                # data = json.loads(line)
                 data_str = line.decode("utf-8")
                 if data_str.startswith("data: "):
                     data = json.loads(data_str[6:])

application/llm/google_ai.py

@@ -0,0 +1,48 @@
+from application.llm.base import BaseLLM
+
+class GoogleLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _clean_messages_google(self, messages):
+        return [
+            {
+                "role": "model" if message["role"] == "system" else message["role"],
+                "parts": [message["content"]],
+            }
+            for message in messages[1:]
+        ]
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        **kwargs
+    ):  
+        import google.generativeai as genai
+        genai.configure(api_key=self.api_key)
+        model = genai.GenerativeModel(model, system_instruction=messages[0]["content"])
+        response = model.generate_content(self._clean_messages_google(messages))
+        return response.text
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        **kwargs
+    ):  
+        import google.generativeai as genai
+        genai.configure(api_key=self.api_key)
+        model = genai.GenerativeModel(model, system_instruction=messages[0]["content"])
+        response = model.generate_content(self._clean_messages_google(messages), stream=True)
+        for line in response:
+            if line.text is not None:
+                yield line.text

application/llm/llm_creator.py

@@ -6,6 +6,7 @@ from application.llm.llama_cpp import LlamaCpp
 from application.llm.anthropic import AnthropicLLM
 from application.llm.docsgpt_provider import DocsGPTAPILLM
 from application.llm.premai import PremAILLM
+from application.llm.google_ai import GoogleLLM
 
 
 class LLMCreator:
@@ -18,7 +19,8 @@ class LLMCreator:
         "anthropic": AnthropicLLM,
         "docsgpt": DocsGPTAPILLM,
         "premai": PremAILLM,
-        "groq": GroqLLM
+        "groq": GroqLLM,
+        "google": GoogleLLM
     }
 
     @classmethod

application/parser/chunking.py

@@ -0,0 +1,118 @@
+import re
+from typing import List, Tuple, Union
+import logging
+from application.parser.schema.base import Document
+from application.utils import get_encoding
+
+logger = logging.getLogger(__name__)
+
+class Chunker:
+    def __init__(
+        self,
+        chunking_strategy: str = "classic_chunk",
+        max_tokens: int = 2000,
+        min_tokens: int = 150,
+        duplicate_headers: bool = False,
+    ):
+        if chunking_strategy not in ["classic_chunk"]:
+            raise ValueError(f"Unsupported chunking strategy: {chunking_strategy}")
+        self.chunking_strategy = chunking_strategy
+        self.max_tokens = max_tokens
+        self.min_tokens = min_tokens
+        self.duplicate_headers = duplicate_headers
+        self.encoding = get_encoding()
+
+    def separate_header_and_body(self, text: str) -> Tuple[str, str]:
+        header_pattern = r"^(.*?\n){3}"
+        match = re.match(header_pattern, text)
+        if match:
+            header = match.group(0)
+            body = text[len(header):]
+        else:
+            header, body = "", text  # No header, treat entire text as body
+        return header, body
+
+    def combine_documents(self, doc: Document, next_doc: Document) -> Document:
+        combined_text = doc.text + " " + next_doc.text
+        combined_token_count = len(self.encoding.encode(combined_text))
+        new_doc = Document(
+            text=combined_text,
+            doc_id=doc.doc_id,
+            embedding=doc.embedding,
+            extra_info={**(doc.extra_info or {}), "token_count": combined_token_count}
+        )
+        return new_doc
+    
+    def split_document(self, doc: Document) -> List[Document]:
+        split_docs = []
+        header, body = self.separate_header_and_body(doc.text)
+        header_tokens = self.encoding.encode(header) if header else []
+        body_tokens = self.encoding.encode(body)
+
+        current_position = 0
+        part_index = 0
+        while current_position < len(body_tokens):
+            end_position = current_position + self.max_tokens - len(header_tokens)
+            chunk_tokens = (header_tokens + body_tokens[current_position:end_position]
+                            if self.duplicate_headers or part_index == 0 else body_tokens[current_position:end_position])
+            chunk_text = self.encoding.decode(chunk_tokens)
+            new_doc = Document(
+                text=chunk_text,
+                doc_id=f"{doc.doc_id}-{part_index}",
+                embedding=doc.embedding,
+                extra_info={**(doc.extra_info or {}), "token_count": len(chunk_tokens)}
+            )
+            split_docs.append(new_doc)
+            current_position = end_position
+            part_index += 1
+            header_tokens = []
+        return split_docs
+
+    def classic_chunk(self, documents: List[Document]) -> List[Document]:
+        processed_docs = []
+        i = 0
+        while i < len(documents):
+            doc = documents[i]
+            tokens = self.encoding.encode(doc.text)
+            token_count = len(tokens)
+
+            if self.min_tokens <= token_count <= self.max_tokens:
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            elif token_count < self.min_tokens:
+                if i + 1 < len(documents):
+                    next_doc = documents[i + 1]
+                    next_tokens = self.encoding.encode(next_doc.text)
+                    if token_count + len(next_tokens) <= self.max_tokens:
+                        # Combine small documents
+                        combined_doc = self.combine_documents(doc, next_doc)
+                        processed_docs.append(combined_doc)
+                        i += 2
+                    else:
+                        # Keep the small document as is if adding next_doc would exceed max_tokens
+                        doc.extra_info = doc.extra_info or {}
+                        doc.extra_info["token_count"] = token_count
+                        processed_docs.append(doc)
+                        i += 1
+                else:
+                    # No next document to combine with; add the small document as is
+                    doc.extra_info = doc.extra_info or {}
+                    doc.extra_info["token_count"] = token_count
+                    processed_docs.append(doc)
+                    i += 1
+            else:
+                # Split large documents
+                processed_docs.extend(self.split_document(doc))
+                i += 1
+        return processed_docs
+
+    def chunk(
+        self,
+        documents: List[Document]
+    ) -> List[Document]:
+        if self.chunking_strategy == "classic_chunk":
+            return self.classic_chunk(documents)
+        else:
+            raise ValueError("Unsupported chunking strategy")

application/parser/embedding_pipeline.py

@@ -0,0 +1,86 @@
+import os
+import logging
+from retry import retry
+from tqdm import tqdm
+from application.core.settings import settings
+from application.vectorstore.vector_creator import VectorCreator
+
+
+@retry(tries=10, delay=60)
+def add_text_to_store_with_retry(store, doc, source_id):
+    """
+    Add a document's text and metadata to the vector store with retry logic.
+    Args:
+        store: The vector store object.
+        doc: The document to be added.
+        source_id: Unique identifier for the source.
+    """
+    try:
+        doc.metadata["source_id"] = str(source_id)
+        store.add_texts([doc.page_content], metadatas=[doc.metadata])
+    except Exception as e:
+        logging.error(f"Failed to add document with retry: {e}")
+        raise
+
+
+def embed_and_store_documents(docs, folder_name, source_id, task_status):
+    """
+    Embeds documents and stores them in a vector store.
+
+    Args:
+        docs (list): List of documents to be embedded and stored.
+        folder_name (str): Directory to save the vector store.
+        source_id (str): Unique identifier for the source.
+        task_status: Task state manager for progress updates.
+
+    Returns:
+        None
+    """
+    # Ensure the folder exists
+    if not os.path.exists(folder_name):
+        os.makedirs(folder_name)
+
+    # Initialize vector store
+    if settings.VECTOR_STORE == "faiss":
+        docs_init = [docs.pop(0)]
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            docs_init=docs_init,
+            source_id=folder_name,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+    else:
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            source_id=source_id,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+        store.delete_index()
+
+    total_docs = len(docs)
+
+    # Process and embed documents
+    for idx, doc in tqdm(
+        docs,
+        desc="Embedding 🦖",
+        unit="docs",
+        total=total_docs,
+        bar_format="{l_bar}{bar}| Time Left: {remaining}",
+    ):
+        try:
+            # Update task status for progress tracking
+            progress = int((idx / total_docs) * 100)
+            task_status.update_state(state="PROGRESS", meta={"current": progress})
+
+            # Add document to vector store
+            add_text_to_store_with_retry(store, doc, source_id)
+        except Exception as e:
+            logging.error(f"Error embedding document {idx}: {e}")
+            logging.info(f"Saving progress at document {idx} out of {total_docs}")
+            store.save_local(folder_name)
+            break
+
+    # Save the vector store
+    if settings.VECTOR_STORE == "faiss":
+        store.save_local(folder_name)
+    logging.info("Vector store saved successfully.")

application/parser/java2doc.py

@@ -1,66 +0,0 @@
-import os
-
-import javalang
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.java'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, "r") as file:
-        java_code = file.read()
-        methods = {}
-        tree = javalang.parse.parse(java_code)
-        for _, node in tree.filter(javalang.tree.MethodDeclaration):
-            method_name = node.name
-            start_line = node.position.line - 1
-            end_line = start_line
-            brace_count = 0
-            for line in java_code.splitlines()[start_line:]:
-                end_line += 1
-                brace_count += line.count("{") - line.count("}")
-                if brace_count == 0:
-                    break
-            method_source_code = "\n".join(java_code.splitlines()[start_line:end_line])
-            methods[method_name] = method_source_code
-    return methods
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = javalang.parse.parse(source_code)
-        for class_decl in tree.types:
-            class_name = class_decl.name
-            declarations = []
-            methods = []
-            for field_decl in class_decl.fields:
-                field_name = field_decl.declarators[0].name
-                field_type = field_decl.type.name
-                declarations.append(f"{field_type} {field_name}")
-            for method_decl in class_decl.methods:
-                methods.append(method_decl.name)
-            class_string = "Declarations: " + ", ".join(declarations) + "\n  Method name: " + ", ".join(methods)
-            classes[class_name] = class_string
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict

application/parser/js2doc.py

@@ -1,70 +0,0 @@
-import os
-
-import escodegen
-import esprima
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.js'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        functions = {}
-        tree = esprima.parseScript(source_code)
-        for node in tree.body:
-            if node.type == 'FunctionDeclaration':
-                func_name = node.id.name if node.id else '<anonymous>'
-                functions[func_name] = escodegen.generate(node)
-            elif node.type == 'VariableDeclaration':
-                for declaration in node.declarations:
-                    if declaration.init and declaration.init.type == 'FunctionExpression':
-                        func_name = declaration.id.name if declaration.id else '<anonymous>'
-                        functions[func_name] = escodegen.generate(declaration.init)
-            elif node.type == 'ClassDeclaration':
-                for subnode in node.body.body:
-                    if subnode.type == 'MethodDefinition':
-                        func_name = subnode.key.name
-                        functions[func_name] = escodegen.generate(subnode.value)
-                    elif subnode.type == 'VariableDeclaration':
-                        for declaration in subnode.declarations:
-                            if declaration.init and declaration.init.type == 'FunctionExpression':
-                                func_name = declaration.id.name if declaration.id else '<anonymous>'
-                                functions[func_name] = escodegen.generate(declaration.init)
-        return functions
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = esprima.parseScript(source_code)
-        for node in tree.body:
-            if node.type == 'ClassDeclaration':
-                class_name = node.id.name
-                function_names = []
-                for subnode in node.body.body:
-                    if subnode.type == 'MethodDefinition':
-                        function_names.append(subnode.key.name)
-                classes[class_name] = ", ".join(function_names)
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict

application/parser/late_chunking.py

@@ -0,0 +1,94 @@
+from typing import List, Tuple, Union, Optional
+from transformers import AutoTokenizer, AutoModel
+from sentence_transformers import SentenceTransformer
+import torch
+import torch.nn as nn
+from application.parser.schema.base import Document
+
+
+class LateChunker:
+    def __init__(self, model_name: str, late_tokens: int = 1000, **model_kwargs):
+        """
+        Initialize the LateChunker with a model, tokenizer, and late_tokens limit.
+        Supports both transformers and sentence-transformers models.
+        """
+        self.late_tokens = late_tokens
+        self.model_name = model_name
+
+        # Load model based on type
+        if "sentence-transformers" in model_name:
+            self.model = SentenceTransformer(model_name, **model_kwargs)
+            self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+            self.wrapper_type = "sentence_transformers"
+        else:
+            self.model = AutoModel.from_pretrained(model_name, trust_remote_code=True, **model_kwargs)
+            self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+            self.wrapper_type = "transformers"
+
+    def tokenize_with_offsets(self, text: str):
+        """Tokenize text and return tokens with character offsets."""
+        tokens = self.tokenizer.encode_plus(
+            text, return_offsets_mapping=True, add_special_tokens=False
+        )
+        return tokens["input_ids"], tokens["offset_mapping"]
+
+    def late_chunk_with_embeddings(
+        self, documents: List[Document]
+    ) -> List[Tuple[str, List[Tuple[int, int]], List[float]]]:
+        """
+        Combines documents into 'super chunks' that fit within `late_tokens` limit.
+        Outputs each super chunk with span annotations and embeddings.
+        """
+        super_chunks = []
+        current_super_chunk_text = []
+        current_token_count = 0
+        span_annotations = []
+
+        for doc in documents:
+            doc_text = doc.text
+            input_ids, offsets = self.tokenize_with_offsets(doc_text)
+            doc_token_count = len(input_ids)
+
+            # Check if adding this document exceeds the late_tokens limit
+            if current_token_count + doc_token_count > self.late_tokens:
+                # Finalize the current super chunk
+                combined_text = " ".join(current_super_chunk_text)
+                embeddings = self.generate_embeddings(combined_text)
+
+                super_chunks.append((combined_text, span_annotations, embeddings))
+
+                # Reset for a new super chunk
+                current_super_chunk_text = []
+                span_annotations = []
+                current_token_count = 0
+
+            # Add document to the current super chunk
+            start_token = current_token_count
+            end_token = current_token_count + doc_token_count
+            span_annotations.append((start_token, end_token))
+            current_super_chunk_text.append(doc_text)
+            current_token_count = end_token
+
+        # Add the final super chunk if there are remaining documents
+        if current_super_chunk_text:
+            combined_text = " ".join(current_super_chunk_text)
+            embeddings = self.generate_embeddings(combined_text)
+            super_chunks.append((combined_text, span_annotations, embeddings))
+
+        return super_chunks
+
+    def generate_embeddings(self, text: str) -> List[float]:
+        """Generate embeddings for a given text using the loaded model."""
+        if self.wrapper_type == "sentence_transformers":
+            # Sentence-Transformers
+            embeddings = self.model.encode([text])
+            return embeddings[0].tolist()
+
+        elif self.wrapper_type == "transformers":
+            # Transformers models
+            inputs = self.tokenizer(text, return_tensors="pt")
+            model_output = self.model(**inputs)
+            return model_output.last_hidden_state.mean(dim=1).squeeze().tolist()
+
+        else:
+            raise ValueError("Unsupported model type for embedding generation.")

application/parser/py2doc.py

@@ -1,121 +0,0 @@
-import ast
-import os
-from pathlib import Path
-
-import tiktoken
-from langchain.llms import OpenAI
-from langchain.prompts import PromptTemplate
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.py'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        functions = {}
-        tree = ast.parse(source_code)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.FunctionDef):
-                func_name = node.name
-                func_def = ast.get_source_segment(source_code, node)
-                functions[func_name] = func_def
-    return functions
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = ast.parse(source_code)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.ClassDef):
-                class_name = node.name
-                function_names = []
-                for subnode in ast.walk(node):
-                    if isinstance(subnode, ast.FunctionDef):
-                        function_names.append(subnode.name)
-                classes[class_name] = ", ".join(function_names)
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict
-
-
-def parse_functions(functions_dict, formats, dir):
-    c1 = len(functions_dict)
-    for i, (source, functions) in enumerate(functions_dict.items(), start=1):
-        print(f"Processing file {i}/{c1}")
-        source_w = source.replace(dir + "/", "").replace("." + formats, ".md")
-        subfolders = "/".join(source_w.split("/")[:-1])
-        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
-        for j, (name, function) in enumerate(functions.items(), start=1):
-            print(f"Processing function {j}/{len(functions)}")
-            prompt = PromptTemplate(
-                input_variables=["code"],
-                template="Code: \n{code}, \nDocumentation: ",
-            )
-            llm = OpenAI(temperature=0)
-            response = llm(prompt.format(code=function))
-            mode = "a" if Path(f"outputs/{source_w}").exists() else "w"
-            with open(f"outputs/{source_w}", mode) as f:
-                f.write(
-                    f"\n\n# Function name: {name} \n\nFunction: \n```\n{function}\n```, \nDocumentation: \n{response}")
-
-
-def parse_classes(classes_dict, formats, dir):
-    c1 = len(classes_dict)
-    for i, (source, classes) in enumerate(classes_dict.items()):
-        print(f"Processing file {i + 1}/{c1}")
-        source_w = source.replace(dir + "/", "").replace("." + formats, ".md")
-        subfolders = "/".join(source_w.split("/")[:-1])
-        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
-        for name, function_names in classes.items():
-            print(f"Processing Class {i + 1}/{c1}")
-            prompt = PromptTemplate(
-                input_variables=["class_name", "functions_names"],
-                template="Class name: {class_name} \nFunctions: {functions_names}, \nDocumentation: ",
-            )
-            llm = OpenAI(temperature=0)
-            response = llm(prompt.format(class_name=name, functions_names=function_names))
-
-            with open(f"outputs/{source_w}", "a" if Path(f"outputs/{source_w}").exists() else "w") as f:
-                f.write(f"\n\n# Class name: {name} \n\nFunctions: \n{function_names}, \nDocumentation: \n{response}")
-
-
-def transform_to_docs(functions_dict, classes_dict, formats, dir):
-    docs_content = ''.join([str(key) + str(value) for key, value in functions_dict.items()])
-    docs_content += ''.join([str(key) + str(value) for key, value in classes_dict.items()])
-
-    num_tokens = len(tiktoken.get_encoding("cl100k_base").encode(docs_content))
-    total_price = ((num_tokens / 1000) * 0.02)
-
-    print(f"Number of Tokens = {num_tokens:,d}")
-    print(f"Approx Cost = ${total_price:,.2f}")
-
-    user_input = input("Price Okay? (Y/N)\n").lower()
-    if user_input == "y" or user_input == "":
-        if not Path("outputs").exists():
-            Path("outputs").mkdir()
-        parse_functions(functions_dict, formats, dir)
-        parse_classes(classes_dict, formats, dir)
-        print("All done!")
-    else:
-        print("The API was not called. No money was spent.")

application/parser/remote/reddit_loader.py

@@ -1,10 +1,19 @@
 from application.parser.remote.base import BaseRemote
 from langchain_community.document_loaders import RedditPostsLoader
+import json
 
 
 class RedditPostsLoaderRemote(BaseRemote):
     def load_data(self, inputs):
-        data = eval(inputs)
+        try:
+            data = json.loads(inputs)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON input: {e}")
+
+        required_fields = ["client_id", "client_secret", "user_agent", "search_queries"]
+        missing_fields = [field for field in required_fields if field not in data]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
         client_id = data.get("client_id")
         client_secret = data.get("client_secret")
         user_agent = data.get("user_agent")

application/retriever/classic_rag.py

@@ -45,7 +45,6 @@ class ClassicRAG(BaseRetriever):
                 settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
             )
             docs_temp = docsearch.search(self.question, k=self.chunks)
-            print(docs_temp)
             docs = [
                 {
                     "title": i.metadata.get(
@@ -60,8 +59,6 @@ class ClassicRAG(BaseRetriever):
                 }
                 for i in docs_temp
             ]
-        if settings.LLM_NAME == "llama.cpp":
-            docs = [docs[0]]
 
         return docs
 

frontend/src/Navigation.tsx

@@ -18,6 +18,7 @@ import SourceDropdown from './components/SourceDropdown';
 import {
   setConversation,
   updateConversationId,
+  handleAbort,
 } from './conversation/conversationSlice';
 import ConversationTile from './conversation/ConversationTile';
 import { useDarkTheme, useMediaQuery, useOutsideAlerter } from './hooks';
@@ -32,7 +33,6 @@ import {
   selectConversations,
   selectModalStateDeleteConv,
   selectSelectedDocs,
-  selectSelectedDocsStatus,
   selectSourceDocs,
   selectPaginatedDocuments,
   setConversations,
@@ -85,10 +85,6 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   const [apiKeyModalState, setApiKeyModalState] =
     useState<ActiveState>('INACTIVE');
 
-  const isSelectedDocsSet = useSelector(selectSelectedDocsStatus);
-  const [selectedDocsModalState, setSelectedDocsModalState] =
-    useState<ActiveState>(isSelectedDocsSet ? 'INACTIVE' : 'ACTIVE');
-
   const [uploadModalState, setUploadModalState] =
     useState<ActiveState>('INACTIVE');
 
@@ -180,6 +176,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   };
 
   const resetConversation = () => {
+    handleAbort();
     dispatch(setConversation([]));
     dispatch(
       updateConversationId({
@@ -491,11 +488,13 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         setModalState={setModalStateDeleteConv}
         handleDeleteAllConv={handleDeleteAllConversations}
       />
-      <Upload
-        modalState={uploadModalState}
-        setModalState={setUploadModalState}
-        isOnboarding={false}
-      ></Upload>
+      {uploadModalState === 'ACTIVE' && (
+        <Upload
+          setModalState={setUploadModalState}
+          isOnboarding={false}
+          close={() => setUploadModalState('INACTIVE')}
+        ></Upload>
+      )}
     </>
   );
 }

frontend/src/components/Dropdown.tsx

@@ -140,7 +140,7 @@ function Dropdown({
                             : option.description
                         }`}
               </span>
-              {showEdit && onEdit && (
+              {showEdit && onEdit && option.type !== 'public' && (
                 <img
                   src={Edit}
                   alt="Edit"

frontend/src/conversation/Conversation.tsx

@@ -1,29 +1,30 @@
 import { Fragment, useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
-import newChatIcon from '../assets/openNewChat.svg';
+import { useNavigate } from 'react-router-dom';
+import Hero from '../Hero';
 import ArrowDown from '../assets/arrow-down.svg';
+import newChatIcon from '../assets/openNewChat.svg';
 import Send from '../assets/send.svg';
 import SendDark from '../assets/send_dark.svg';
+import ShareIcon from '../assets/share.svg';
 import SpinnerDark from '../assets/spinner-dark.svg';
 import Spinner from '../assets/spinner.svg';
 import RetryIcon from '../components/RetryIcon';
-import { useNavigate } from 'react-router-dom';
-import Hero from '../Hero';
 import { useDarkTheme, useMediaQuery } from '../hooks';
 import { ShareConversationModal } from '../modals/ShareConversationModal';
-import { setConversation, updateConversationId } from './conversationSlice';
 import { selectConversationId } from '../preferences/preferenceSlice';
 import { AppDispatch } from '../store';
 import ConversationBubble from './ConversationBubble';
 import { handleSendFeedback } from './conversationHandlers';
 import { FEEDBACK, Query } from './conversationModels';
-import ShareIcon from '../assets/share.svg';
 import {
   addQuery,
   fetchAnswer,
   selectQueries,
   selectStatus,
+  setConversation,
+  updateConversationId,
   updateQuery,
 } from './conversationSlice';
 
@@ -302,14 +303,14 @@ export default function Conversation() {
         )}
       </div>
 
-      <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 pb-1 sm:w-[62%] h-auto">
+      <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 z-3 sm:w-[62%] h-auto">
         <div className="flex w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black">
           <textarea
             id="inputbox"
             ref={inputRef}
             tabIndex={1}
             placeholder={t('inputPlaceholder')}
-            className={`inputbox-style h-16 w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap rounded-full bg-white pt-5 pb-[22px] text-base leading-tight opacity-100 focus:outline-none dark:bg-raisin-black dark:text-bright-gray`}
+            className={`inputbox-style w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap rounded-full bg-transparent py-5 text-base leading-tight opacity-100 focus:outline-none dark:bg-transparent dark:text-bright-gray`}
             onInput={handleInput}
             onKeyDown={(e) => {
               if (e.key === 'Enter' && !e.shiftKey) {

frontend/src/conversation/ConversationBubble.tsx

@@ -1,12 +1,14 @@
+import 'katex/dist/katex.min.css';
+
 import { forwardRef, useState } from 'react';
 import ReactMarkdown from 'react-markdown';
 import { useSelector } from 'react-redux';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { vscDarkPlus } from 'react-syntax-highlighter/dist/cjs/styles/prism';
+import rehypeKatex from 'rehype-katex';
 import remarkGfm from 'remark-gfm';
 import remarkMath from 'remark-math';
-import rehypeKatex from 'rehype-katex';
-import 'katex/dist/katex.min.css';
+
 import DocsGPT3 from '../assets/cute_docsgpt3.svg';
 import Dislike from '../assets/dislike.svg?react';
 import Document from '../assets/document.svg';
@@ -16,13 +18,13 @@ import Sources from '../assets/sources.svg';
 import Avatar from '../components/Avatar';
 import CopyButton from '../components/CopyButton';
 import Sidebar from '../components/Sidebar';
+import SpeakButton from '../components/TextToSpeechButton';
 import {
   selectChunks,
   selectSelectedDocs,
 } from '../preferences/preferenceSlice';
 import classes from './ConversationBubble.module.css';
 import { FEEDBACK, MESSAGE_TYPE } from './conversationModels';
-import SpeakButton from '../components/TextToSpeechButton';
 
 const DisableSourceFE = import.meta.env.VITE_DISABLE_SOURCE_FE || false;
 
@@ -41,6 +43,7 @@ const ConversationBubble = forwardRef<
   { message, type, className, feedback, handleFeedback, sources, retryBtn },
   ref,
 ) {
+  // const bubbleRef = useRef<HTMLDivElement | null>(null);
   const chunks = useSelector(selectChunks);
   const selectedDocs = useSelector(selectSelectedDocs);
   const [isLikeHovered, setIsLikeHovered] = useState(false);
@@ -141,7 +144,7 @@ const ConversationBubble = forwardRef<
                 />
                 <p className="text-base font-semibold">Sources</p>
               </div>
-              <div className="ml-3 mr-5 max-w-[90vw] md:max-w-[70vw] lg:max-w-[50vw]">
+              <div className="fade-in ml-3 mr-5 max-w-[90vw] md:max-w-[70vw] lg:max-w-[50vw]">
                 <div className="grid grid-cols-2 gap-2 lg:grid-cols-4">
                   {sources?.slice(0, 3)?.map((source, index) => (
                     <div key={index} className="relative">
@@ -190,7 +193,7 @@ const ConversationBubble = forwardRef<
                       </div>
                       {activeTooltip === index && (
                         <div
-                          className={`absolute left-1/2 z-30 max-h-48 w-40 translate-x-[-50%] translate-y-[3px] rounded-xl bg-[#FBFBFB] p-4 text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver sm:w-56`}
+                          className={`absolute left-1/2 z-50 max-h-48 w-40 translate-x-[-50%] translate-y-[3px] rounded-xl bg-[#FBFBFB] p-4 text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver sm:w-56`}
                           onMouseOver={() => setActiveTooltip(index)}
                           onMouseOut={() => setActiveTooltip(null)}
                         >
@@ -231,14 +234,14 @@ const ConversationBubble = forwardRef<
             <p className="text-base font-semibold">Answer</p>
           </div>
           <div
-            className={`ml-2 mr-5 flex max-w-[90vw] rounded-[28px] bg-gray-1000 py-[14px] px-7 dark:bg-gun-metal md:max-w-[70vw] lg:max-w-[50vw] ${
+            className={`fade-in-bubble ml-2 mr-5 flex max-w-[90vw] rounded-[28px] bg-gray-1000 py-[14px] px-7 dark:bg-gun-metal md:max-w-[70vw] lg:max-w-[50vw] ${
               type === 'ERROR'
                 ? 'relative flex-row items-center rounded-full border border-transparent bg-[#FFE7E7] p-2 py-5 text-sm font-normal text-red-3000  dark:border-red-2000 dark:text-white'
                 : 'flex-col rounded-3xl'
             }`}
           >
             <ReactMarkdown
-              className="whitespace-pre-wrap break-normal leading-normal"
+              className="fade-in whitespace-pre-wrap break-normal leading-normal"
               remarkPlugins={[remarkGfm, remarkMath]}
               rehypePlugins={[rehypeKatex]}
               components={{

frontend/src/conversation/conversationSlice.ts

@@ -17,9 +17,23 @@ const initialState: ConversationState = {
 
 const API_STREAMING = import.meta.env.VITE_API_STREAMING === 'true';
 
+let abortController: AbortController | null = null;
+export function handleAbort() {
+  if (abortController) {
+    abortController.abort();
+    abortController = null;
+  }
+}
+
 export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
   'fetchAnswer',
-  async ({ question }, { dispatch, getState, signal }) => {
+  async ({ question }, { dispatch, getState }) => {
+    if (abortController) {
+      abortController.abort();
+    }
+    abortController = new AbortController();
+    const { signal } = abortController;
+
     let isSourceUpdated = false;
     const state = getState() as RootState;
     if (state.preference) {

frontend/src/index.css

@@ -514,3 +514,29 @@ input:-webkit-autofill:focus {
 .logs-table {
   font-family: 'IBMPlexMono-Medium', system-ui;
 }
+
+.fade-in {
+  animation: fadeIn 0.5s ease-in-out;
+}
+
+@keyframes fadeIn {
+  0% {
+    opacity: 0;
+  }
+  100% {
+    opacity: 1;
+  }
+}
+
+.fade-in-bubble {
+  opacity: 0;
+  transform: translateY(10px);
+  animation: fadeInUp 0.5s forwards;
+}
+
+@keyframes fadeInUp {
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}

frontend/src/modals/ConfirmationModal.tsx

@@ -1,6 +1,6 @@
-import Exit from '../assets/exit.svg';
 import { ActiveState } from '../models/misc';
 import { useTranslation } from 'react-i18next';
+import WrapperModal from './WrapperModal';
 function ConfirmationModal({
   message,
   modalState,
@@ -20,49 +20,43 @@ function ConfirmationModal({
 }) {
   const { t } = useTranslation();
   return (
-    <article
-      className={`${
-        modalState === 'ACTIVE' ? 'visible' : 'hidden'
-      } fixed top-0 left-0 z-30  h-screen w-screen  bg-gray-alpha`}
-    >
-      <article className="mx-auto mt-[35vh] flex w-[90vw] max-w-lg  flex-col gap-4 rounded-2xl bg-white shadow-lg dark:bg-outer-space">
-        <div className="relative">
-          <button
-            className="absolute top-3 right-4 m-2 w-3"
-            onClick={() => {
-              setModalState('INACTIVE');
-              handleCancel && handleCancel();
-            }}
-          >
-            <img className="filter dark:invert" src={Exit} />
-          </button>
-          <div className="p-8">
-            <p className="font-base mb-1 w-[90%] text-lg text-jet dark:text-bright-gray">
-              {message}
-            </p>
-            <div>
-              <div className="mt-6 flex flex-row-reverse gap-1">
-                <button
-                  onClick={handleSubmit}
-                  className="rounded-3xl bg-purple-30 px-5 py-2 text-sm text-white transition-all hover:bg-[#6F3FD1]"
-                >
-                  {submitLabel}
-                </button>
-                <button
-                  onClick={() => {
-                    setModalState('INACTIVE');
-                    handleCancel && handleCancel();
-                  }}
-                  className="cursor-pointer rounded-3xl px-5 py-2 text-sm font-medium hover:bg-gray-100 dark:bg-transparent dark:text-light-gray dark:hover:bg-[#767183]/50"
-                >
-                  {cancelLabel ? cancelLabel : t('cancel')}
-                </button>
+    <>
+      {modalState === 'ACTIVE' && (
+        <WrapperModal
+          close={() => {
+            setModalState('INACTIVE');
+            handleCancel && handleCancel();
+          }}
+        >
+          <div className="relative">
+            <div className="p-8">
+              <p className="font-base mb-1 w-[90%] text-lg text-jet dark:text-bright-gray">
+                {message}
+              </p>
+              <div>
+                <div className="mt-6 flex flex-row-reverse gap-1">
+                  <button
+                    onClick={handleSubmit}
+                    className="rounded-3xl bg-purple-30 px-5 py-2 text-sm text-white transition-all hover:bg-[#6F3FD1]"
+                  >
+                    {submitLabel}
+                  </button>
+                  <button
+                    onClick={() => {
+                      setModalState('INACTIVE');
+                      handleCancel && handleCancel();
+                    }}
+                    className="cursor-pointer rounded-3xl px-5 py-2 text-sm font-medium hover:bg-gray-100 dark:bg-transparent dark:text-light-gray dark:hover:bg-[#767183]/50"
+                  >
+                    {cancelLabel ? cancelLabel : t('cancel')}
+                  </button>
+                </div>
               </div>
             </div>
           </div>
-        </div>
-      </article>
-    </article>
+        </WrapperModal>
+      )}
+    </>
   );
 }
 

frontend/src/modals/CreateAPIKeyModal.tsx

@@ -3,11 +3,11 @@ import { useTranslation } from 'react-i18next';
 import { useSelector } from 'react-redux';
 
 import userService from '../api/services/userService';
-import Exit from '../assets/exit.svg';
 import Dropdown from '../components/Dropdown';
 import Input from '../components/Input';
 import { CreateAPIKeyModalProps, Doc } from '../models/misc';
 import { selectSourceDocs } from '../preferences/preferenceSlice';
+import WrapperModal from './WrapperModal';
 
 const embeddingsName =
   import.meta.env.VITE_EMBEDDINGS_NAME ||
@@ -73,91 +73,82 @@ export default function CreateAPIKeyModal({
     handleFetchPrompts();
   }, []);
   return (
-    <div className="fixed top-0 left-0 z-30 flex h-screen w-screen items-center justify-center bg-gray-alpha bg-opacity-50">
-      <div className="relative w-11/12 rounded-2xl bg-white p-10 dark:bg-outer-space sm:w-[512px]">
-        <button className="absolute top-3 right-4 m-2 w-3" onClick={close}>
-          <img className="filter dark:invert" src={Exit} />
-        </button>
-        <div className="mb-6">
-          <span className="text-xl text-jet dark:text-bright-gray">
-            {t('modals.createAPIKey.label')}
-          </span>
-        </div>
-        <div className="relative mt-5 mb-4">
-          <span className="absolute left-2 -top-2 bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
-            {t('modals.createAPIKey.apiKeyName')}
-          </span>
-          <Input
-            type="text"
-            className="rounded-md"
-            value={APIKeyName}
-            onChange={(e) => setAPIKeyName(e.target.value)}
-          ></Input>
-        </div>
-        <div className="my-4">
-          <Dropdown
-            placeholder={t('modals.createAPIKey.sourceDoc')}
-            selectedValue={sourcePath ? sourcePath.name : null}
-            onSelect={(selection: {
-              name: string;
-              id: string;
-              type: string;
-            }) => {
-              setSourcePath(selection);
-            }}
-            options={extractDocPaths()}
-            size="w-full"
-            rounded="xl"
-            border="border"
-          />
-        </div>
-        <div className="my-4">
-          <Dropdown
-            options={activePrompts}
-            selectedValue={prompt ? prompt.name : null}
-            placeholder={t('modals.createAPIKey.prompt')}
-            onSelect={(value: { name: string; id: string; type: string }) =>
-              setPrompt(value)
-            }
-            size="w-full"
-            border="border"
-          />
-        </div>
-        <div className="my-4">
-          <p className="mb-2 ml-2 font-semibold text-jet dark:text-bright-gray">
-            {t('modals.createAPIKey.chunks')}
-          </p>
-          <Dropdown
-            options={chunkOptions}
-            selectedValue={chunk}
-            onSelect={(value: string) => setChunk(value)}
-            size="w-full"
-            border="border"
-          />
-        </div>
-        <button
-          disabled={!sourcePath || APIKeyName.length === 0 || !prompt}
-          onClick={() => {
-            if (sourcePath && prompt) {
-              const payload: any = {
-                name: APIKeyName,
-                prompt_id: prompt.id,
-                chunks: chunk,
-              };
-              if (sourcePath.type === 'default') {
-                payload.retriever = sourcePath.id;
-              }
-              if (sourcePath.type === 'local') {
-                payload.source = sourcePath.id;
-              }
-              createAPIKey(payload);
-            }
-          }}
-          className="float-right mt-4 rounded-full bg-purple-30 px-5 py-2 text-sm text-white hover:bg-[#6F3FD1] disabled:opacity-50"
-        >
-          {t('modals.createAPIKey.create')}
-        </button>
+    <WrapperModal close={close}>
+      <div className="mb-6">
+        <span className="text-xl text-jet dark:text-bright-gray">
+          {t('modals.createAPIKey.label')}
+        </span>
       </div>
-    </div>
+      <div className="relative mt-5 mb-4">
+        <span className="absolute left-2 -top-2 bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+          {t('modals.createAPIKey.apiKeyName')}
+        </span>
+        <Input
+          type="text"
+          className="rounded-md"
+          value={APIKeyName}
+          onChange={(e) => setAPIKeyName(e.target.value)}
+        ></Input>
+      </div>
+      <div className="my-4">
+        <Dropdown
+          placeholder={t('modals.createAPIKey.sourceDoc')}
+          selectedValue={sourcePath ? sourcePath.name : null}
+          onSelect={(selection: { name: string; id: string; type: string }) => {
+            setSourcePath(selection);
+          }}
+          options={extractDocPaths()}
+          size="w-full"
+          rounded="xl"
+          border="border"
+        />
+      </div>
+      <div className="my-4">
+        <Dropdown
+          options={activePrompts}
+          selectedValue={prompt ? prompt.name : null}
+          placeholder={t('modals.createAPIKey.prompt')}
+          onSelect={(value: { name: string; id: string; type: string }) =>
+            setPrompt(value)
+          }
+          size="w-full"
+          border="border"
+        />
+      </div>
+      <div className="my-4">
+        <p className="mb-2 ml-2 font-semibold text-jet dark:text-bright-gray">
+          {t('modals.createAPIKey.chunks')}
+        </p>
+        <Dropdown
+          options={chunkOptions}
+          selectedValue={chunk}
+          onSelect={(value: string) => setChunk(value)}
+          size="w-full"
+          border="border"
+        />
+      </div>
+      <button
+        disabled={!sourcePath || APIKeyName.length === 0 || !prompt}
+        onClick={() => {
+          if (sourcePath && prompt) {
+            const payload: any = {
+              name: APIKeyName,
+              prompt_id: prompt.id,
+              chunks: chunk,
+            };
+            if (sourcePath.type === 'default') {
+              payload.retriever = sourcePath.id;
+            }
+            if (sourcePath.type === 'local') {
+              payload.source = sourcePath.id;
+            }
+            createAPIKey(payload);
+          }
+        }}
+        className="float-right mt-4 rounded-full bg-purple-30 px-5 py-2 text-sm text-white hover:bg-[#6F3FD1] disabled:opacity-50"
+      >
+        {t('modals.createAPIKey.create')}
+      </button>
+    </WrapperModal>
   );
 }

frontend/src/modals/ShareConversationModal.tsx

@@ -10,7 +10,6 @@ import {
 import Dropdown from '../components/Dropdown';
 import { Doc } from '../models/misc';
 import Spinner from '../assets/spinner.svg';
-import Exit from '../assets/exit.svg';
 const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
 const embeddingsName =
   import.meta.env.VITE_EMBEDDINGS_NAME ||
@@ -19,6 +18,7 @@ const embeddingsName =
 type StatusType = 'loading' | 'idle' | 'fetched' | 'failed';
 
 import conversationService from '../api/services/conversationService';
+import WrapperModal from './WrapperModal';
 
 export const ShareConversationModal = ({
   close,
@@ -99,85 +99,78 @@ export const ShareConversationModal = ({
   };
 
   return (
-    <div className="fixed top-0 left-0 z-40 flex h-screen w-screen cursor-default items-center justify-center bg-gray-alpha bg-opacity-50 text-chinese-black dark:text-silver">
-      <div className="relative w-11/12 rounded-2xl bg-white p-10 dark:bg-outer-space sm:w-[512px]">
-        <button className="absolute top-3 right-4 m-2 w-3" onClick={close}>
-          <img className="filter dark:invert" src={Exit} />
-        </button>
-        <div className="flex flex-col gap-2">
-          <h2 className="text-xl font-medium">{t('modals.shareConv.label')}</h2>
-          <p className="text-sm">{t('modals.shareConv.note')}</p>
-          <div className="flex items-center justify-between">
-            <span className="text-lg">{t('modals.shareConv.option')}</span>
-            <label className=" cursor-pointer select-none items-center">
-              <div className="relative">
-                <input
-                  type="checkbox"
-                  checked={allowPrompt}
-                  onChange={togglePromptPermission}
-                  className="sr-only"
-                />
-                <div
-                  className={`box block h-8 w-14 rounded-full border border-purple-30 ${
-                    allowPrompt
-                      ? 'bg-purple-30 dark:bg-purple-30'
-                      : 'dark:bg-transparent'
-                  }`}
-                ></div>
-                <div
-                  className={`absolute left-1 top-1 flex h-6 w-6 items-center justify-center rounded-full  transition ${
-                    allowPrompt ? 'translate-x-full bg-silver' : 'bg-purple-30'
-                  }`}
-                ></div>
-              </div>
-            </label>
-          </div>
-          {allowPrompt && (
-            <div className="my-4">
-              <Dropdown
-                placeholder={t('modals.createAPIKey.sourceDoc')}
-                selectedValue={sourcePath}
-                onSelect={(selection: { label: string; value: string }) =>
-                  setSourcePath(selection)
-                }
-                options={extractDocPaths(sourceDocs ?? [])}
-                size="w-full"
-                rounded="xl"
+    <WrapperModal close={close}>
+      <div className="flex flex-col gap-2">
+        <h2 className="text-xl font-medium">{t('modals.shareConv.label')}</h2>
+        <p className="text-sm">{t('modals.shareConv.note')}</p>
+        <div className="flex items-center justify-between">
+          <span className="text-lg">{t('modals.shareConv.option')}</span>
+          <label className=" cursor-pointer select-none items-center">
+            <div className="relative">
+              <input
+                type="checkbox"
+                checked={allowPrompt}
+                onChange={togglePromptPermission}
+                className="sr-only"
               />
+              <div
+                className={`box block h-8 w-14 rounded-full border border-purple-30 ${
+                  allowPrompt
+                    ? 'bg-purple-30 dark:bg-purple-30'
+                    : 'dark:bg-transparent'
+                }`}
+              ></div>
+              <div
+                className={`absolute left-1 top-1 flex h-6 w-6 items-center justify-center rounded-full  transition ${
+                  allowPrompt ? 'translate-x-full bg-silver' : 'bg-purple-30'
+                }`}
+              ></div>
             </div>
-          )}
-          <div className="flex items-baseline justify-between gap-2">
-            <span className="no-scrollbar w-full overflow-x-auto whitespace-nowrap rounded-full border-2 py-3 px-4">
-              {`${domain}/share/${identifier ?? '....'}`}
-            </span>
-            {status === 'fetched' ? (
-              <button
-                className="my-1 h-10 w-28 rounded-full border border-solid  bg-purple-30 p-2 text-sm text-white hover:bg-[#6F3FD1]"
-                onClick={() => handleCopyKey(`${domain}/share/${identifier}`)}
-              >
-                {isCopied
-                  ? t('modals.saveKey.copied')
-                  : t('modals.saveKey.copy')}
-              </button>
-            ) : (
-              <button
-                className="my-1 flex h-10 w-28 items-center justify-evenly rounded-full  border border-solid bg-purple-30 p-2 text-center text-sm font-normal text-white hover:bg-[#6F3FD1]"
-                onClick={() => {
-                  shareCoversationPublicly(allowPrompt);
-                }}
-              >
-                {t('modals.shareConv.create')}
-                {status === 'loading' && (
-                  <img
-                    src={Spinner}
-                    className="inline animate-spin cursor-pointer bg-transparent filter dark:invert"
-                  ></img>
-                )}
-              </button>
-            )}
+          </label>
+        </div>
+        {allowPrompt && (
+          <div className="my-4">
+            <Dropdown
+              placeholder={t('modals.createAPIKey.sourceDoc')}
+              selectedValue={sourcePath}
+              onSelect={(selection: { label: string; value: string }) =>
+                setSourcePath(selection)
+              }
+              options={extractDocPaths(sourceDocs ?? [])}
+              size="w-full"
+              rounded="xl"
+            />
           </div>
+        )}
+        <div className="flex items-baseline justify-between gap-2">
+          <span className="no-scrollbar w-full overflow-x-auto whitespace-nowrap rounded-full border-2 py-3 px-4">
+            {`${domain}/share/${identifier ?? '....'}`}
+          </span>
+          {status === 'fetched' ? (
+            <button
+              className="my-1 h-10 w-28 rounded-full border border-solid  bg-purple-30 p-2 text-sm text-white hover:bg-[#6F3FD1]"
+              onClick={() => handleCopyKey(`${domain}/share/${identifier}`)}
+            >
+              {isCopied ? t('modals.saveKey.copied') : t('modals.saveKey.copy')}
+            </button>
+          ) : (
+            <button
+              className="my-1 flex h-10 w-28 items-center justify-evenly rounded-full  border border-solid bg-purple-30 p-2 text-center text-sm font-normal text-white hover:bg-[#6F3FD1]"
+              onClick={() => {
+                shareCoversationPublicly(allowPrompt);
+              }}
+            >
+              {t('modals.shareConv.create')}
+              {status === 'loading' && (
+                <img
+                  src={Spinner}
+                  className="inline animate-spin cursor-pointer bg-transparent filter dark:invert"
+                ></img>
+              )}
+            </button>
+          )}
         </div>
       </div>
-    </div>
+    </WrapperModal>
   );
 };

frontend/src/modals/WrapperModal.tsx

@@ -0,0 +1,55 @@
+import React, { useEffect, useRef } from 'react';
+import { WrapperModalProps } from './types';
+import Exit from '../assets/exit.svg';
+
+const WrapperModal: React.FC<WrapperModalProps> = ({
+  children,
+  close,
+  isPerformingTask,
+}) => {
+  const modalRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (isPerformingTask) return;
+    const handleClickOutside = (event: MouseEvent) => {
+      if (
+        modalRef.current &&
+        !modalRef.current.contains(event.target as Node)
+      ) {
+        close();
+      }
+    };
+
+    const handleEscapePress = (event: KeyboardEvent) => {
+      if (event.key === 'Escape') {
+        close();
+      }
+    };
+
+    document.addEventListener('mousedown', handleClickOutside);
+    document.addEventListener('keydown', handleEscapePress);
+
+    return () => {
+      document.removeEventListener('mousedown', handleClickOutside);
+      document.removeEventListener('keydown', handleEscapePress);
+    };
+  }, [close]);
+
+  return (
+    <div className="fixed top-0 left-0 z-30 flex h-screen w-screen items-center justify-center bg-gray-alpha bg-opacity-50">
+      <div
+        ref={modalRef}
+        className="relative w-11/12 rounded-2xl bg-white p-10 dark:bg-outer-space sm:w-[512px]"
+      >
+        {!isPerformingTask && (
+          <button className="absolute top-3 right-4 m-2 w-3" onClick={close}>
+            <img className="filter dark:invert" src={Exit} />
+          </button>
+        )}
+        {children}
+      </div>
+    </div>
+  );
+};
+
+export default WrapperModal;

frontend/src/modals/types/index.ts

@@ -0,0 +1,5 @@
+export type WrapperModalProps = {
+  children?: React.ReactNode;
+  isPerformingTask?: boolean;
+  close: () => void;
+};

frontend/src/preferences/PromptsModal.tsx

@@ -1,6 +1,7 @@
 import { ActiveState } from '../models/misc';
 import Exit from '../assets/exit.svg';
 import Input from '../components/Input';
+import React from 'react';
 
 function AddPrompt({
   setModalState,
@@ -9,6 +10,7 @@ function AddPrompt({
   setNewPromptName,
   newPromptContent,
   setNewPromptContent,
+  disableSave,
 }: {
   setModalState: (state: ActiveState) => void;
   handleAddPrompt?: () => void;
@@ -16,6 +18,7 @@ function AddPrompt({
   setNewPromptName: (name: string) => void;
   newPromptContent: string;
   setNewPromptContent: (content: string) => void;
+  disableSave: boolean;
 }) {
   return (
     <div className="relative">
@@ -23,6 +26,8 @@ function AddPrompt({
         className="absolute top-3 right-4 m-2 w-3"
         onClick={() => {
           setModalState('INACTIVE');
+          setNewPromptName('');
+          setNewPromptContent('');
         }}
       >
         <img className="filter dark:invert" src={Exit} />
@@ -41,7 +46,7 @@ function AddPrompt({
             className="h-10 rounded-lg"
             value={newPromptName}
             onChange={(e) => setNewPromptName(e.target.value)}
-          ></Input>
+          />
           <div className="relative bottom-12 left-3 mt-[-3.00px]">
             <span className="bg-white px-1 text-xs text-silver dark:bg-outer-space dark:text-silver">
               Prompt Name
@@ -62,6 +67,8 @@ function AddPrompt({
           <button
             onClick={handleAddPrompt}
             className="rounded-3xl bg-purple-30 px-5 py-2 text-sm text-white transition-all hover:opacity-90"
+            disabled={disableSave}
+            title={disableSave && newPromptName ? 'Name already exists' : ''}
           >
             Save
           </button>
@@ -79,6 +86,7 @@ function EditPrompt({
   editPromptContent,
   setEditPromptContent,
   currentPromptEdit,
+  disableSave,
 }: {
   setModalState: (state: ActiveState) => void;
   handleEditPrompt?: (id: string, type: string) => void;
@@ -87,6 +95,7 @@ function EditPrompt({
   editPromptContent: string;
   setEditPromptContent: (content: string) => void;
   currentPromptEdit: { name: string; id: string; type: string };
+  disableSave: boolean;
 }) {
   return (
     <div className="relative">
@@ -140,7 +149,8 @@ function EditPrompt({
               handleEditPrompt &&
                 handleEditPrompt(currentPromptEdit.id, currentPromptEdit.type);
             }}
-            disabled={currentPromptEdit.type === 'public'}
+            disabled={currentPromptEdit.type === 'public' || disableSave}
+            title={disableSave && editPromptName ? 'Name already exists' : ''}
           >
             Save
           </button>
@@ -151,6 +161,7 @@ function EditPrompt({
 }
 
 export default function PromptsModal({
+  existingPrompts,
   modalState,
   setModalState,
   type,
@@ -166,6 +177,7 @@ export default function PromptsModal({
   handleAddPrompt,
   handleEditPrompt,
 }: {
+  existingPrompts: { name: string; id: string; type: string }[];
   modalState: ActiveState;
   setModalState: (state: ActiveState) => void;
   type: 'ADD' | 'EDIT';
@@ -181,6 +193,25 @@ export default function PromptsModal({
   handleAddPrompt?: () => void;
   handleEditPrompt?: (id: string, type: string) => void;
 }) {
+  const [disableSave, setDisableSave] = React.useState(true);
+  const handlePrompNameChange = (edit: boolean, newName: string) => {
+    const nameExists = existingPrompts.find(
+      (prompt) => newName === prompt.name,
+    );
+
+    if (newName && !nameExists) {
+      setDisableSave(false);
+    } else {
+      setDisableSave(true);
+    }
+
+    if (edit) {
+      setEditPromptName(newName);
+    } else {
+      setNewPromptName(newName);
+    }
+  };
+
   let view;
 
   if (type === 'ADD') {
@@ -189,9 +220,10 @@ export default function PromptsModal({
         setModalState={setModalState}
         handleAddPrompt={handleAddPrompt}
         newPromptName={newPromptName}
-        setNewPromptName={setNewPromptName}
+        setNewPromptName={handlePrompNameChange.bind(null, false)}
         newPromptContent={newPromptContent}
         setNewPromptContent={setNewPromptContent}
+        disableSave={disableSave}
       />
     );
   } else if (type === 'EDIT') {
@@ -200,10 +232,11 @@ export default function PromptsModal({
         setModalState={setModalState}
         handleEditPrompt={handleEditPrompt}
         editPromptName={editPromptName}
-        setEditPromptName={setEditPromptName}
+        setEditPromptName={handlePrompNameChange.bind(null, true)}
         editPromptContent={editPromptContent}
         setEditPromptContent={setEditPromptContent}
         currentPromptEdit={currentPromptEdit}
+        disableSave={disableSave}
       />
     );
   } else {

frontend/src/settings/Documents.tsx

@@ -255,9 +255,9 @@ const Documents: React.FC<DocumentsProps> = ({
             <div className="w-full h-full bg-transparent flex flex-col items-center justify-center p-8">
               {/* Your Upload component */}
               <Upload
-                modalState={modalState}
                 setModalState={setModalState}
                 isOnboarding={isOnboarding}
+                close={() => setModalState('INACTIVE')}
               />
             </div>
           </div>

frontend/src/settings/Prompts.tsx

@@ -58,7 +58,8 @@ export default function Prompts({
       }
       setModalState('INACTIVE');
       onSelectPrompt(newPromptName, newPrompt.id, newPromptContent);
-      setNewPromptName(newPromptName);
+      setNewPromptName('');
+      setNewPromptContent('');
     } catch (error) {
       console.error(error);
     }
@@ -178,6 +179,7 @@ export default function Prompts({
         </div>
       </div>
       <PromptsModal
+        existingPrompts={prompts}
         type={modalType}
         modalState={modalState}
         setModalState={setModalState}

frontend/src/upload/Upload.tsx

@@ -4,7 +4,6 @@ import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
 
 import userService from '../api/services/userService';
-import Exit from '../assets/exit.svg';
 import ArrowLeft from '../assets/arrow-left.svg';
 import FileUpload from '../assets/file_upload.svg';
 import WebsiteCollect from '../assets/website_collect.svg';
@@ -17,15 +16,16 @@ import {
   setSourceDocs,
   selectSourceDocs,
 } from '../preferences/preferenceSlice';
+import WrapperModal from '../modals/WrapperModal';
 
 function Upload({
-  modalState,
   setModalState,
   isOnboarding,
+  close,
 }: {
-  modalState: ActiveState;
   setModalState: (state: ActiveState) => void;
   isOnboarding: boolean;
+  close: () => void;
 }) {
   const [docName, setDocName] = useState('');
   const [urlName, setUrlName] = useState('');
@@ -76,7 +76,7 @@ function Upload({
         <div className="relative w-32 h-32 rounded-full">
           <div className="absolute inset-0 rounded-full shadow-[0_0_10px_2px_rgba(0,0,0,0.3)_inset] dark:shadow-[0_0_10px_2px_rgba(0,0,0,0.3)_inset]"></div>
           <div
-            className={`absolute inset-0 rounded-full ${progressPercent === 100 ? 'shadow-xl shadow-lime-300/50 dark:shadow-lime-300/50 bg-gradient-to-r from-white to-gray-400 dark:bg-gradient-to-br dark:from-gray-500 dark:to-gray-300' : 'shadow-[0_2px_0_#FF3D00_inset] dark:shadow-[0_2px_0_#FF3D00_inset]'}`}
+            className={`absolute inset-0 rounded-full ${progressPercent === 100 ? 'shadow-xl shadow-lime-300/50 dark:shadow-lime-300/50 bg-gradient-to-r from-white to-gray-400 dark:bg-gradient-to-br dark:from-gray-500 dark:to-gray-300' : 'shadow-[0_4px_0_#7D54D1] dark:shadow-[0_4px_0_#7D54D1]'}`}
             style={{
               animation: `${progressPercent === 100 ? 'none' : 'rotate 2s linear infinite'}`,
             }}
@@ -600,7 +600,30 @@ function Upload({
             ) : (
               <button
                 onClick={uploadRemote}
-                className={`ml-2 cursor-pointer rounded-3xl bg-purple-30 py-2 px-6 text-sm text-white hover:bg-[#6F3FD1]`}
+                className={`ml-2 cursor-pointer rounded-3xl bg-purple-30 py-2 px-6 text-sm text-white hover:bg-[#6F3FD1] ${
+                  urlName.trim().length === 0 ||
+                  url.trim().length === 0 ||
+                  (urlType.label === 'Reddit' &&
+                    (redditData.client_id.length === 0 ||
+                      redditData.client_secret.length === 0 ||
+                      redditData.user_agent.length === 0 ||
+                      redditData.search_queries.length === 0 ||
+                      redditData.number_posts === 0)) ||
+                  (urlType.label === 'GitHub' && repoUrl.trim().length === 0)
+                    ? 'bg-opacity-80 text-opacity-80'
+                    : ''
+                }`}
+                disabled={
+                  urlName.trim().length === 0 ||
+                  url.trim().length === 0 ||
+                  (urlType.label === 'Reddit' &&
+                    (redditData.client_id.length === 0 ||
+                      redditData.client_secret.length === 0 ||
+                      redditData.user_agent.length === 0 ||
+                      redditData.search_queries.length === 0 ||
+                      redditData.number_posts === 0)) ||
+                  (urlType.label === 'GitHub' && repoUrl.trim().length === 0)
+                }
               >
                 {t('modals.uploadDoc.train')}
               </button>
@@ -626,28 +649,18 @@ function Upload({
   }
 
   return (
-    <article
-      className={`${
-        modalState === 'ACTIVE' ? 'visible' : 'hidden'
-      } absolute z-30 bg-gray-alpha flex items-center justify-center h-[calc(100vh-4rem)] md:h-screen w-full`}
+    <WrapperModal
+      isPerformingTask={progress !== undefined && progress.percentage < 100}
+      close={() => {
+        close();
+        setDocName('');
+        setfiles([]);
+        setModalState('INACTIVE');
+        setActiveTab(null);
+      }}
     >
-      <article className="relative mx-auto flex w-[90vw] max-w-lg  flex-col gap-4 rounded-lg bg-white p-6 shadow-lg dark:bg-outer-space h-fit-content">
-        {!isOnboarding && !progress && (
-          <button
-            className="absolute top-4 right-4 m-1 w-3"
-            onClick={() => {
-              setDocName('');
-              setfiles([]);
-              setModalState('INACTIVE');
-              setActiveTab(null);
-            }}
-          >
-            <img className="filter dark:invert" src={Exit} />
-          </button>
-        )}
-        {view}
-      </article>
-    </article>
+      {view}
+    </WrapperModal>
   );
 }
 

package.json

@@ -1,7 +0,0 @@
-{
-  "devDependencies": {
-    "eslint": "^8.53.0",
-    "lint-staged": "^15.1.0",
-    "prettier": "^3.1.0"
-  }
-}