Update README.md

Feature/upload exit btn

.github/workflows/cife.yml

@@ -0,0 +1,44 @@
+name: Build and push DocsGPT-FE Docker image
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          
+      - name: Login to ghcr.io
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      # Runs a single command using the runners shell
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v2
+        with:
+          file: './frontend/Dockerfile'
+          platforms: linux/amd64
+          context: ./frontend
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest

.gitignore

@@ -162,3 +162,11 @@ frontend/*.sw?
 application/vectors/
 
 **/inputs
+
+**/indexes
+
+**/temp
+
+**/yarn.lock
+
+node_modules/

README.md

@@ -21,6 +21,11 @@ Say goodbye to time-consuming manual searches, and let <strong>DocsGPT</strong>
   
 </div>
 
+![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
+
+
+## Features
+
 ![Group 9](https://user-images.githubusercontent.com/17906039/220427472-2644cff4-7666-46a5-819f-fc4a521f63c7.png)
 
 
@@ -29,8 +34,7 @@ Say goodbye to time-consuming manual searches, and let <strong>DocsGPT</strong>
 
 You can find our [Roadmap](https://github.com/orgs/arc53/projects/2) here, please don't hesitate contributing or creating issues, it helps us make DocsGPT better!
 
-## Preview
-![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov2.gif)
+
 
 ## [Live preview](https://docsgpt.arc53.com/)
 
@@ -44,8 +48,23 @@ You can find our [Roadmap](https://github.com/orgs/arc53/projects/2) here, pleas
 
 - Scripts - script that creates similarity search index and store for other libraries. 
 
+- frontend - frontend in vite and
+
 ## QuickStart
-Please note: current vector database uses pandas Python documentation, thus responses will be related to it, if you want to use other docs please follow a guide below
+
+Note: Make sure you have docker installed
+
+1. Open dowload this repository with `git clone https://github.com/arc53/DocsGPT.git`
+2. Open docker-compose.yaml and replace <your_api_key> with your OpenAI's key (there are 4 places)
+3. Run `docker-compose build && docker-compose up`
+
+To stop just run Ctrl + C
+
+## Development environments
+
+Spin up only 2 containers from docker-compose.yaml (by deleting all services except for redis and mongo)
+
+Make sure you have python 3.10 or 3.11 installed
 
 1. Navigate to `/application` folder
 2. Install dependencies
@@ -54,6 +73,15 @@ Please note: current vector database uses pandas Python documentation, thus resp
 Copy .env_sample and create .env with your openai api token
 4. Run the app
 `python app.py`
+5. Start worker with `celery -A app.celery worker -l INFO`
+
+To start frontend
+1. Navigate to `/frontend` folder
+2. Install dependencies
+`npm install`
+3. In the file  `.env.development` instead of `VITE_API_HOST = https://docsapi.arc53.com` use `VITE_API_HOST=http://localhost:5001`
+3. Run the app
+4. `npm run dev`
 
 
 [How to install the Chrome extension](https://github.com/arc53/docsgpt/wiki#launch-chrome-extension)

application/.env_sample

@@ -1 +1,6 @@
-OPENAI_API_KEY=your_api_key
+API_KEY=your_api_key
+EMBEDDINGS_KEY=your_api_key
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/1
+MONGO_URI=mongodb://localhost:27017/docsgpt
+API_URL=http://localhost:5001

application/Dockerfile

@@ -5,16 +5,21 @@ RUN apt-get update && apt-get install -y gcc curl
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && apt-get install --reinstall libc6-dev -y
 ENV PATH="/root/.cargo/bin:${PATH}"
 RUN pip install --upgrade pip && pip install tiktoken==0.1.2
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
 
 FROM python:3.10-slim-bullseye
 # Copy pre-built packages from builder stage
 COPY --from=builder /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/
+RUN pip install gunicorn==20.1.0
+RUN pip install celery==5.2.7
 WORKDIR /app
 COPY . /app
 ENV FLASK_APP=app.py
 ENV FLASK_DEBUG=true
-RUN pip install -r requirements.txt
+
 
 EXPOSE 5001
 
-CMD ["gunicorn", "-w", "6", "--bind", "0.0.0.0:5001", "wsgi:app"]
+CMD ["gunicorn", "-w", "2", "--timeout", "120", "--bind", "0.0.0.0:5001", "wsgi:app"]

application/app.py

@@ -1,25 +1,38 @@
-import os
+import datetime
 import json
+import os
 import traceback
 
 import dotenv
 import requests
-from flask import Flask, request, render_template
+from celery import Celery
+from celery.result import AsyncResult
+from flask import Flask, request, render_template, send_from_directory, jsonify
 from langchain import FAISS
-from langchain.llms import OpenAIChat
 from langchain import VectorDBQA, HuggingFaceHub, Cohere, OpenAI
+from langchain.chains import ChatVectorDBChain
 from langchain.chains.question_answering import load_qa_chain
+from langchain.chat_models import ChatOpenAI
 from langchain.embeddings import OpenAIEmbeddings, HuggingFaceHubEmbeddings, CohereEmbeddings, \
     HuggingFaceInstructEmbeddings
 from langchain.prompts import PromptTemplate
-from error import bad_request
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    SystemMessagePromptTemplate,
+    HumanMessagePromptTemplate,
+)
+from pymongo import MongoClient
+from werkzeug.utils import secure_filename
 
-os.environ["LANGCHAIN_HANDLER"] = "langchain"
+from error import bad_request
+from worker import ingest_worker
+
+# os.environ["LANGCHAIN_HANDLER"] = "langchain"
 
 if os.getenv("LLM_NAME") is not None:
     llm_choice = os.getenv("LLM_NAME")
 else:
-    llm_choice = "openai"
+    llm_choice = "openai_chat"
 
 if os.getenv("EMBEDDINGS_NAME") is not None:
     embeddings_choice = os.getenv("EMBEDDINGS_NAME")
@@ -47,15 +60,22 @@ if platform.system() == "Windows":
 # loading the .env file
 dotenv.load_dotenv()
 
-with open("combine_prompt.txt", "r") as f:
+# load the prompts
+with open("prompts/combine_prompt.txt", "r") as f:
     template = f.read()
 
-with open("combine_prompt_hist.txt", "r") as f:
+with open("prompts/combine_prompt_hist.txt", "r") as f:
     template_hist = f.read()
 
-with open("question_prompt.txt", "r") as f:
+with open("prompts/question_prompt.txt", "r") as f:
     template_quest = f.read()
 
+with open("prompts/chat_combine_prompt.txt", "r") as f:
+    chat_combine_template = f.read()
+
+with open("prompts/chat_reduce_prompt.txt", "r") as f:
+    chat_reduce_template = f.read()
+
 if os.getenv("API_KEY") is not None:
     api_key_set = True
 else:
@@ -66,6 +86,21 @@ else:
     embeddings_key_set = False
 
 app = Flask(__name__)
+app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER = "inputs"
+app.config['CELERY_BROKER_URL'] = os.getenv("CELERY_BROKER_URL")
+app.config['CELERY_RESULT_BACKEND'] = os.getenv("CELERY_RESULT_BACKEND")
+app.config['MONGO_URI'] = os.getenv("MONGO_URI")
+celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'], backend=app.config['CELERY_RESULT_BACKEND'])
+celery.conf.update(app.config)
+mongo = MongoClient(app.config['MONGO_URI'])
+db = mongo["docsgpt"]
+vectors_collection = db["vectors"]
+
+
+@celery.task(bind=True)
+def ingest(self, directory, formats, name_job, filename, user):
+    resp = ingest_worker(self, directory, formats, name_job, filename, user)
+    return resp
 
 
 @app.route("/")
@@ -93,12 +128,16 @@ def api_answer():
     try:
         # check if the vectorstore is set
         if "active_docs" in data:
-            vectorstore = "vectors/" + data["active_docs"]
+            if data["active_docs"].split("/")[0] == "local":
+                vectorstore = "indexes/" + data["active_docs"]
+            else:
+                vectorstore = "vectors/" + data["active_docs"]
             if data['active_docs'] == "default":
                 vectorstore = ""
         else:
             vectorstore = ""
-        #vectorstore = "outputs/inputs/"
+        print(vectorstore)
+        # vectorstore = "outputs/inputs/"
         # loading the index and the store and the prompt template
         # Note if you have used other embeddings than OpenAI, you need to change the embeddings
         if embeddings_choice == "openai_text-embedding-ada-002":
@@ -123,9 +162,21 @@ def api_answer():
 
         q_prompt = PromptTemplate(input_variables=["context", "question"], template=template_quest,
                                   template_format="jinja2")
-        if llm_choice == "openai":
-            llm = OpenAIChat(openai_api_key=api_key, temperature=0)
-            #llm = OpenAI(openai_api_key=api_key, temperature=0)
+        if llm_choice == "openai_chat":
+            # llm = ChatOpenAI(openai_api_key=api_key, model_name="gpt-4")
+            llm = ChatOpenAI(openai_api_key=api_key)
+            messages_combine = [
+                SystemMessagePromptTemplate.from_template(chat_combine_template),
+                HumanMessagePromptTemplate.from_template("{question}")
+            ]
+            p_chat_combine = ChatPromptTemplate.from_messages(messages_combine)
+            messages_reduce = [
+                SystemMessagePromptTemplate.from_template(chat_reduce_template),
+                HumanMessagePromptTemplate.from_template("{question}")
+            ]
+            p_chat_reduce = ChatPromptTemplate.from_messages(messages_reduce)
+        elif llm_choice == "openai":
+            llm = OpenAI(openai_api_key=api_key, temperature=0)
         elif llm_choice == "manifest":
             llm = ManifestWrapper(client=manifest, llm_kwargs={"temperature": 0.001, "max_tokens": 2048})
         elif llm_choice == "huggingface":
@@ -133,16 +184,26 @@ def api_answer():
         elif llm_choice == "cohere":
             llm = Cohere(model="command-xlarge-nightly", cohere_api_key=api_key)
 
-        qa_chain = load_qa_chain(llm=llm, chain_type="map_reduce",
-                                 combine_prompt=c_prompt, question_prompt=q_prompt)
+        if llm_choice == "openai_chat":
+            chain = ChatVectorDBChain.from_llm(
+                llm=llm,
+                vectorstore=docsearch,
+                prompt=p_chat_combine,
+                qa_prompt=p_chat_reduce,
+                top_k_docs_for_context=3,
+                return_source_documents=False)
+            result = chain({"question": question, "chat_history": []})
+        else:
+            qa_chain = load_qa_chain(llm=llm, chain_type="map_reduce",
+                                     combine_prompt=c_prompt, question_prompt=q_prompt)
+            chain = VectorDBQA(combine_documents_chain=qa_chain, vectorstore=docsearch, k=4)
+            result = chain({"query": question})
 
-        chain = VectorDBQA(combine_documents_chain=qa_chain, vectorstore=docsearch, k=10)
-
-        # fetch the answer
-        result = chain({"query": question})
+        print(result)
 
         # some formatting for the frontend
-        result['answer'] = result['result']
+        if "result" in result:
+            result['answer'] = result['result']
         result['answer'] = result['answer'].replace("\\n", "\n")
         try:
             result['answer'] = result['answer'].split("SOURCES:")[0]
@@ -166,6 +227,9 @@ def api_answer():
 def check_docs():
     # check if docs exist in a vectorstore folder
     data = request.get_json()
+    # split docs on / and take first part
+    if data["docs"].split("/")[0] == "local":
+        return {"status": 'exists'}
     vectorstore = "vectors/" + data["docs"]
     base_path = 'https://raw.githubusercontent.com/arc53/DocsHUB/main/'
     if os.path.exists(vectorstore) or data["docs"] == "default":
@@ -189,12 +253,184 @@ def check_docs():
         return {"status": 'loaded'}
 
 
+@app.route("/api/feedback", methods=["POST"])
+def api_feedback():
+    data = request.get_json()
+    question = data["question"]
+    answer = data["answer"]
+    feedback = data["feedback"]
+
+    print('-' * 5)
+    print("Question: " + question)
+    print("Answer: " + answer)
+    print("Feedback: " + feedback)
+    print('-' * 5)
+    response = requests.post(
+        url="https://86x89umx77.execute-api.eu-west-2.amazonaws.com/docsgpt-feedback",
+        headers={
+            "Content-Type": "application/json; charset=utf-8",
+        },
+        data=json.dumps({
+            "answer": answer,
+            "question": question,
+            "feedback": feedback
+        })
+    )
+    return {"status": 'ok'}
+
+
+@app.route('/api/combine', methods=['GET'])
+def combined_json():
+    user = 'local'
+    """Provide json file with combined available indexes."""
+    # get json from https://d3dg1063dc54p9.cloudfront.net/combined.json
+
+    data = []
+    # structure: name, language, version, description, fullName, date, docLink
+    # append data from vectors_collection
+    for index in vectors_collection.find({'user': user}):
+        data.append({
+            "name": index['name'],
+            "language": index['language'],
+            "version": '',
+            "description": index['name'],
+            "fullName": index['name'],
+            "date": index['date'],
+            "docLink": index['location'],
+            "model": embeddings_choice,
+            "location": "local"
+        })
+
+    data_remote = requests.get("https://d3dg1063dc54p9.cloudfront.net/combined.json").json()
+    for index in data_remote:
+        index['location'] = "remote"
+        data.append(index)
+
+    return jsonify(data)
+
+
+@app.route('/api/upload', methods=['POST'])
+def upload_file():
+    """Upload a file to get vectorized and indexed."""
+    if 'user' not in request.form:
+        return {"status": 'no user'}
+    user = secure_filename(request.form['user'])
+    if 'name' not in request.form:
+        return {"status": 'no name'}
+    job_name = secure_filename(request.form['name'])
+    # check if the post request has the file part
+    if 'file' not in request.files:
+        print('No file part')
+        return {"status": 'no file'}
+    file = request.files['file']
+    if file.filename == '':
+        return {"status": 'no file name'}
+
+    if file:
+        filename = secure_filename(file.filename)
+        # save dir
+        save_dir = os.path.join(app.config['UPLOAD_FOLDER'], user, job_name)
+        # create dir if not exists
+        if not os.path.exists(save_dir):
+            os.makedirs(save_dir)
+
+        file.save(os.path.join(save_dir, filename))
+        task = ingest.delay('temp', [".rst", ".md", ".pdf"], job_name, filename, user)
+        # task id
+        task_id = task.id
+        return {"status": 'ok', "task_id": task_id}
+    else:
+        return {"status": 'error'}
+
+
+@app.route('/api/task_status', methods=['GET'])
+def task_status():
+    """Get celery job status."""
+    task_id = request.args.get('task_id')
+    task = AsyncResult(task_id)
+    task_meta = task.info
+    return {"status": task.status, "result": task_meta}
+
+
+### Backgound task api
+@app.route('/api/upload_index', methods=['POST'])
+def upload_index_files():
+    """Upload two files(index.faiss, index.pkl) to the user's folder."""
+    if 'user' not in request.form:
+        return {"status": 'no user'}
+    user = secure_filename(request.form['user'])
+    if 'name' not in request.form:
+        return {"status": 'no name'}
+    job_name = secure_filename(request.form['name'])
+    if 'file_faiss' not in request.files:
+        print('No file part')
+        return {"status": 'no file'}
+    file_faiss = request.files['file_faiss']
+    if file_faiss.filename == '':
+        return {"status": 'no file name'}
+    if 'file_pkl' not in request.files:
+        print('No file part')
+        return {"status": 'no file'}
+    file_pkl = request.files['file_pkl']
+    if file_pkl.filename == '':
+        return {"status": 'no file name'}
+
+    # saves index files
+    save_dir = os.path.join('indexes', user, job_name)
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+    file_faiss.save(os.path.join(save_dir, 'index.faiss'))
+    file_pkl.save(os.path.join(save_dir, 'index.pkl'))
+    # create entry in vectors_collection
+    vectors_collection.insert_one({
+        "user": user,
+        "name": job_name,
+        "language": job_name,
+        "location": save_dir,
+        "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
+        "model": embeddings_choice,
+        "type": "local"
+    })
+    return {"status": 'ok'}
+
+
+@app.route('/api/download', methods=['get'])
+def download_file():
+    user = secure_filename(request.args.get('user'))
+    job_name = secure_filename(request.args.get('name'))
+    filename = secure_filename(request.args.get('file'))
+    save_dir = os.path.join(app.config['UPLOAD_FOLDER'], user, job_name)
+    return send_from_directory(save_dir, filename, as_attachment=True)
+
+
+@app.route('/api/delete_old', methods=['get'])
+def delete_old():
+    """Delete old indexes."""
+    import shutil
+    path = request.args.get('path')
+    dirs = path.split('/')
+    dirs_clean = []
+    for i in range(1, len(dirs)):
+        dirs_clean.append(secure_filename(dirs[i]))
+    # check that path strats with indexes or vectors
+    if dirs[0] not in ['indexes', 'vectors']:
+        return {"status": 'error'}
+    path_clean = '/'.join(dirs)
+    vectors_collection.delete_one({'location': path})
+    try:
+        shutil.rmtree(path_clean)
+    except FileNotFoundError:
+        pass
+    return {"status": 'ok'}
+
+
 # handling CORS
 @app.after_request
 def after_request(response):
     response.headers.add('Access-Control-Allow-Origin', '*')
     response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
     response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
+    response.headers.add('Access-Control-Allow-Credentials', 'true')
     return response
 
 

application/parser/__init__.py

@@ -0,0 +1 @@
+

application/parser/file/base.py

@@ -0,0 +1,20 @@
+"""Base reader class."""
+from abc import abstractmethod
+from typing import Any, List
+
+from langchain.docstore.document import Document as LCDocument
+
+from parser.schema.base import Document
+
+
+class BaseReader:
+    """Utilities for loading data from a directory."""
+
+    @abstractmethod
+    def load_data(self, *args: Any, **load_kwargs: Any) -> List[Document]:
+        """Load data from the input directory."""
+
+    def load_langchain_documents(self, **load_kwargs: Any) -> List[LCDocument]:
+        """Load data in LangChain document format."""
+        docs = self.load_data(**load_kwargs)
+        return [d.to_langchain_format() for d in docs]

application/parser/file/base_parser.py

@@ -0,0 +1,38 @@
+"""Base parser and config class."""
+
+from abc import abstractmethod
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+
+class BaseParser:
+    """Base class for all parsers."""
+
+    def __init__(self, parser_config: Optional[Dict] = None):
+        """Init params."""
+        self._parser_config = parser_config
+
+    def init_parser(self) -> None:
+        """Init parser and store it."""
+        parser_config = self._init_parser()
+        self._parser_config = parser_config
+
+    @property
+    def parser_config_set(self) -> bool:
+        """Check if parser config is set."""
+        return self._parser_config is not None
+
+    @property
+    def parser_config(self) -> Dict:
+        """Check if parser config is set."""
+        if self._parser_config is None:
+            raise ValueError("Parser config not set.")
+        return self._parser_config
+
+    @abstractmethod
+    def _init_parser(self) -> Dict:
+        """Initialize the parser with the config."""
+
+    @abstractmethod
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""

application/parser/file/bulk.py

@@ -0,0 +1,163 @@
+"""Simple reader that reads files of different formats from a directory."""
+import logging
+from pathlib import Path
+from typing import Callable, Dict, List, Optional, Union
+
+from parser.file.base import BaseReader
+from parser.file.base_parser import BaseParser
+from parser.file.docs_parser import DocxParser, PDFParser
+from parser.file.epub_parser import EpubParser
+from parser.file.html_parser import HTMLParser
+from parser.file.markdown_parser import MarkdownParser
+from parser.file.rst_parser import RstParser
+from parser.file.tabular_parser import PandasCSVParser
+from parser.schema.base import Document
+
+DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
+    ".pdf": PDFParser(),
+    ".docx": DocxParser(),
+    ".csv": PandasCSVParser(),
+    ".epub": EpubParser(),
+    ".md": MarkdownParser(),
+    ".rst": RstParser(),
+    ".html": HTMLParser(),
+    ".mdx": MarkdownParser(),
+}
+
+
+class SimpleDirectoryReader(BaseReader):
+    """Simple directory reader.
+
+    Can read files into separate documents, or concatenates
+    files into one document text.
+
+    Args:
+        input_dir (str): Path to the directory.
+        input_files (List): List of file paths to read (Optional; overrides input_dir)
+        exclude_hidden (bool): Whether to exclude hidden files (dotfiles).
+        errors (str): how encoding and decoding errors are to be handled,
+              see https://docs.python.org/3/library/functions.html#open
+        recursive (bool): Whether to recursively search in subdirectories.
+            False by default.
+        required_exts (Optional[List[str]]): List of required extensions.
+            Default is None.
+        file_extractor (Optional[Dict[str, BaseParser]]): A mapping of file
+            extension to a BaseParser class that specifies how to convert that file
+            to text. See DEFAULT_FILE_EXTRACTOR.
+        num_files_limit (Optional[int]): Maximum number of files to read.
+            Default is None.
+        file_metadata (Optional[Callable[str, Dict]]): A function that takes
+            in a filename and returns a Dict of metadata for the Document.
+            Default is None.
+    """
+
+    def __init__(
+        self,
+        input_dir: Optional[str] = None,
+        input_files: Optional[List] = None,
+        exclude_hidden: bool = True,
+        errors: str = "ignore",
+        recursive: bool = True,
+        required_exts: Optional[List[str]] = None,
+        file_extractor: Optional[Dict[str, BaseParser]] = None,
+        num_files_limit: Optional[int] = None,
+        file_metadata: Optional[Callable[[str], Dict]] = None,
+        chunk_size_max: int = 2048,
+    ) -> None:
+        """Initialize with parameters."""
+        super().__init__()
+
+        if not input_dir and not input_files:
+            raise ValueError("Must provide either `input_dir` or `input_files`.")
+
+        self.errors = errors
+
+        self.recursive = recursive
+        self.exclude_hidden = exclude_hidden
+        self.required_exts = required_exts
+        self.num_files_limit = num_files_limit
+
+        if input_files:
+            self.input_files = []
+            for path in input_files:
+                print(path)
+                input_file = Path(path)
+                self.input_files.append(input_file)
+        elif input_dir:
+            self.input_dir = Path(input_dir)
+            self.input_files = self._add_files(self.input_dir)
+
+        self.file_extractor = file_extractor or DEFAULT_FILE_EXTRACTOR
+        self.file_metadata = file_metadata
+
+    def _add_files(self, input_dir: Path) -> List[Path]:
+        """Add files."""
+        input_files = sorted(input_dir.iterdir())
+        new_input_files = []
+        dirs_to_explore = []
+        for input_file in input_files:
+            if input_file.is_dir():
+                if self.recursive:
+                    dirs_to_explore.append(input_file)
+            elif self.exclude_hidden and input_file.name.startswith("."):
+                continue
+            elif (
+                self.required_exts is not None
+                and input_file.suffix not in self.required_exts
+            ):
+                continue
+            else:
+                new_input_files.append(input_file)
+
+        for dir_to_explore in dirs_to_explore:
+            sub_input_files = self._add_files(dir_to_explore)
+            new_input_files.extend(sub_input_files)
+
+        if self.num_files_limit is not None and self.num_files_limit > 0:
+            new_input_files = new_input_files[0 : self.num_files_limit]
+
+        # print total number of files added
+        logging.debug(
+            f"> [SimpleDirectoryReader] Total files added: {len(new_input_files)}"
+        )
+
+        return new_input_files
+
+    def load_data(self, concatenate: bool = False) -> List[Document]:
+        """Load data from the input directory.
+
+        Args:
+            concatenate (bool): whether to concatenate all files into one document.
+                If set to True, file metadata is ignored.
+                False by default.
+
+        Returns:
+            List[Document]: A list of documents.
+
+        """
+        data: Union[str, List[str]] = ""
+        data_list: List[str] = []
+        metadata_list = []
+        for input_file in self.input_files:
+            if input_file.suffix in self.file_extractor:
+                parser = self.file_extractor[input_file.suffix]
+                if not parser.parser_config_set:
+                    parser.init_parser()
+                data = parser.parse_file(input_file, errors=self.errors)
+            else:
+                # do standard read
+                with open(input_file, "r", errors=self.errors) as f:
+                    data = f.read()
+            if isinstance(data, List):
+                data_list.extend(data)
+            else:
+                data_list.append(str(data))
+            if self.file_metadata is not None:
+                metadata_list.append(self.file_metadata(str(input_file)))
+
+        if concatenate:
+            return [Document("\n".join(data_list))]
+        elif self.file_metadata is not None:
+            return [Document(d, extra_info=m) for d, m in zip(data_list, metadata_list)]
+        else:
+            return [Document(d) for d in data_list]

application/parser/file/docs_parser.py

@@ -0,0 +1,59 @@
+"""Docs parser.
+
+Contains parsers for docx, pdf files.
+
+"""
+from pathlib import Path
+from typing import Dict
+
+from parser.file.base_parser import BaseParser
+
+
+class PDFParser(BaseParser):
+    """PDF parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        """Parse file."""
+        try:
+            import PyPDF2
+        except ImportError:
+            raise ValueError("PyPDF2 is required to read PDF files.")
+        text_list = []
+        with open(file, "rb") as fp:
+            # Create a PDF object
+            pdf = PyPDF2.PdfReader(fp)
+
+            # Get the number of pages in the PDF document
+            num_pages = len(pdf.pages)
+
+            # Iterate over every page
+            for page in range(num_pages):
+                # Extract the text from the page
+                page_text = pdf.pages[page].extract_text()
+                text_list.append(page_text)
+        text = "\n".join(text_list)
+
+        return text
+
+
+class DocxParser(BaseParser):
+    """Docx parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        """Parse file."""
+        try:
+            import docx2txt
+        except ImportError:
+            raise ValueError("docx2txt is required to read Microsoft Word files.")
+
+        text = docx2txt.process(file)
+
+        return text

application/parser/file/epub_parser.py

@@ -0,0 +1,43 @@
+"""Epub parser.
+
+Contains parsers for epub files.
+"""
+
+from pathlib import Path
+from typing import Dict
+
+from parser.file.base_parser import BaseParser
+
+
+class EpubParser(BaseParser):
+    """Epub Parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        """Parse file."""
+        try:
+            import ebooklib
+            from ebooklib import epub
+        except ImportError:
+            raise ValueError("`EbookLib` is required to read Epub files.")
+        try:
+            import html2text
+        except ImportError:
+            raise ValueError("`html2text` is required to parse Epub files.")
+
+        text_list = []
+        book = epub.read_epub(file, options={"ignore_ncx": True})
+
+        # Iterate through all chapters.
+        for item in book.get_items():
+            # Chapters are typically located in epub documents items.
+            if item.get_type() == ebooklib.ITEM_DOCUMENT:
+                text_list.append(
+                    html2text.html2text(item.get_content().decode("utf-8"))
+                )
+
+        text = "\n".join(text_list)
+        return text

application/parser/file/html_parser.py

@@ -0,0 +1,82 @@
+"""HTML parser.
+
+Contains parser for html files.
+
+"""
+import re
+from pathlib import Path
+from typing import Dict, Union
+
+from parser.file.base_parser import BaseParser
+
+class HTMLParser(BaseParser):
+    """HTML parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        """Parse file.
+
+            Returns:
+            Union[str, List[str]]: a string or a List of strings.
+        """
+        try:
+            import unstructured
+        except ImportError:
+            raise ValueError("unstructured package is required to parse HTML files.")
+        from unstructured.partition.html import partition_html
+        from unstructured.staging.base import convert_to_isd
+        from unstructured.cleaners.core import clean
+
+        # Using the unstructured library to convert the html to isd format
+        # isd sample : isd = [
+                            #   {"text": "My Title", "type": "Title"},
+                            #   {"text": "My Narrative", "type": "NarrativeText"}
+                            # ]
+        with open(file, "r", encoding="utf-8") as fp:
+            elements = partition_html(file=fp)
+            isd = convert_to_isd(elements)  
+
+        # Removing non ascii charactwers from isd_el['text']
+        for isd_el in isd:
+            isd_el['text'] = isd_el['text'].encode("ascii", "ignore").decode()
+
+        # Removing all the \n characters from isd_el['text'] using regex and replace with single space
+        # Removing all the extra spaces  from isd_el['text'] using regex and replace with single space
+        for isd_el in isd:
+            isd_el['text'] = re.sub(r'\n', ' ', isd_el['text'], flags=re.MULTILINE|re.DOTALL)
+            isd_el['text'] = re.sub(r"\s{2,}"," ", isd_el['text'], flags=re.MULTILINE|re.DOTALL)
+
+        # more cleaning: extra_whitespaces, dashes, bullets, trailing_punctuation
+        for isd_el in isd:
+            clean(isd_el['text'], extra_whitespace=True, dashes=True, bullets=True, trailing_punctuation=True )
+
+        # Creating a list of all the indexes of isd_el['type'] = 'Title'
+        title_indexes = [i for i,isd_el in enumerate(isd) if isd_el['type'] == 'Title']
+
+        # Creating 'Chunks' - List of lists of strings 
+        # each list starting with with isd_el['type'] = 'Title' and all the data till the next 'Title'
+        # Each Chunk can be thought of as an individual set of data, which can be sent to the model
+        # Where Each Title is grouped together with the data under it
+
+        Chunks = [[]]
+        final_chunks = list(list())
+
+        for i,isd_el in enumerate(isd):
+            if i in title_indexes:
+                Chunks.append([])
+            Chunks[-1].append(isd_el['text'])
+
+        # Removing all the chunks with sum of lenth of all the strings in the chunk < 25 #TODO: This value can be an user defined variable
+        for chunk in Chunks:
+            # sum of lenth of all the strings in the chunk
+            sum = 0
+            sum += len(str(chunk))
+            if sum < 25:
+                Chunks.remove(chunk)
+            else :         
+                # appending all the approved chunks to final_chunks as a single string       
+                final_chunks.append(" ".join([str(item) for item in chunk]))
+        return final_chunks

application/parser/file/markdown_parser.py

@@ -0,0 +1,144 @@
+"""Markdown parser.
+
+Contains parser for md files.
+
+"""
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
+
+from parser.file.base_parser import BaseParser
+import tiktoken
+
+
+class MarkdownParser(BaseParser):
+    """Markdown parser.
+
+    Extract text from markdown files.
+    Returns dictionary with keys as headers and values as the text between headers.
+
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        remove_hyperlinks: bool = True,
+        remove_images: bool = True,
+        max_tokens: int = 2048,
+        # remove_tables: bool = True,
+        **kwargs: Any,
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._remove_hyperlinks = remove_hyperlinks
+        self._remove_images = remove_images
+        self._max_tokens = max_tokens
+        # self._remove_tables = remove_tables
+
+
+    def tups_chunk_append(self, tups: List[Tuple[Optional[str], str]], current_header: Optional[str], current_text: str):
+        """Append to tups chunk."""
+        num_tokens = len(tiktoken.get_encoding("cl100k_base").encode(current_text))
+        if num_tokens > self._max_tokens:
+            chunks = [current_text[i:i + self._max_tokens] for i in range(0, len(current_text), self._max_tokens)]
+            for chunk in chunks:
+                tups.append((current_header, chunk))
+        else:
+            tups.append((current_header, current_text))
+        return tups
+    def markdown_to_tups(self, markdown_text: str) -> List[Tuple[Optional[str], str]]:
+        """Convert a markdown file to a dictionary.
+
+        The keys are the headers and the values are the text under each header.
+
+        """
+        markdown_tups: List[Tuple[Optional[str], str]] = []
+        lines = markdown_text.split("\n")
+
+        current_header = None
+        current_text = ""
+
+        for line in lines:
+            header_match = re.match(r"^#+\s", line)
+            if header_match:
+                if current_header is not None:
+                    if current_text == "" or None:
+                        continue
+                    markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)
+
+                current_header = line
+                current_text = ""
+            else:
+                current_text += line + "\n"
+        markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)
+
+        if current_header is not None:
+            # pass linting, assert keys are defined
+            markdown_tups = [
+                (re.sub(r"#", "", cast(str, key)).strip(), re.sub(r"<.*?>", "", value))
+                for key, value in markdown_tups
+            ]
+        else:
+            markdown_tups = [
+                (key, re.sub("\n", "", value)) for key, value in markdown_tups
+            ]
+
+        return markdown_tups
+
+    def remove_images(self, content: str) -> str:
+        """Get a dictionary of a markdown file from its path."""
+        pattern = r"!{1}\[\[(.*)\]\]"
+        content = re.sub(pattern, "", content)
+        return content
+
+    # def remove_tables(self, content: str) -> List[List[str]]:
+    #     """Convert markdown tables to nested lists."""
+    #     table_rows_pattern = r"((\r?\n){2}|^)([^\r\n]*\|[^\r\n]*(\r?\n)?)+(?=(\r?\n){2}|$)"
+    #     table_cells_pattern = r"([^\|\r\n]*)\|"
+    #
+    #     table_rows = re.findall(table_rows_pattern, content, re.MULTILINE)
+    #     table_lists = []
+    #     for row in table_rows:
+    #         cells = re.findall(table_cells_pattern, row[2])
+    #         cells = [cell.strip() for cell in cells if cell.strip()]
+    #         table_lists.append(cells)
+    #     return str(table_lists)
+
+    def remove_hyperlinks(self, content: str) -> str:
+        """Get a dictionary of a markdown file from its path."""
+        pattern = r"\[(.*?)\]\((.*?)\)"
+        content = re.sub(pattern, r"\1", content)
+        return content
+
+    def _init_parser(self) -> Dict:
+        """Initialize the parser with the config."""
+        return {}
+
+    def parse_tups(
+        self, filepath: Path, errors: str = "ignore"
+    ) -> List[Tuple[Optional[str], str]]:
+        """Parse file into tuples."""
+        with open(filepath, "r") as f:
+            content = f.read()
+        if self._remove_hyperlinks:
+            content = self.remove_hyperlinks(content)
+        if self._remove_images:
+            content = self.remove_images(content)
+        # if self._remove_tables:
+        #     content = self.remove_tables(content)
+        markdown_tups = self.markdown_to_tups(content)
+        return markdown_tups
+
+    def parse_file(
+        self, filepath: Path, errors: str = "ignore"
+    ) -> Union[str, List[str]]:
+        """Parse file into string."""
+        tups = self.parse_tups(filepath, errors=errors)
+        results = []
+        # TODO: don't include headers right now
+        for header, value in tups:
+            if header is None:
+                results.append(value)
+            else:
+                results.append(f"\n\n{header}\n{value}")
+        return results

application/parser/file/rst_parser.py

@@ -0,0 +1,173 @@
+"""reStructuredText parser.
+
+Contains parser for md files.
+
+"""
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
+
+from parser.file.base_parser import BaseParser
+import tiktoken
+
+class RstParser(BaseParser):
+    """reStructuredText parser.
+
+    Extract text from .rst files.
+    Returns dictionary with keys as headers and values as the text between headers.
+
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        remove_hyperlinks: bool = True,
+        remove_images: bool = True,
+        remove_table_excess: bool = True,
+        remove_interpreters: bool = True,
+        remove_directives: bool = True,
+        remove_whitespaces_excess: bool = True,
+        #Be carefull with remove_characters_excess, might cause data loss
+        remove_characters_excess: bool = True,
+        **kwargs: Any,
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._remove_hyperlinks = remove_hyperlinks
+        self._remove_images = remove_images
+        self._remove_table_excess = remove_table_excess
+        self._remove_interpreters = remove_interpreters
+        self._remove_directives = remove_directives
+        self._remove_whitespaces_excess = remove_whitespaces_excess
+        self._remove_characters_excess = remove_characters_excess
+
+
+    def rst_to_tups(self, rst_text: str) -> List[Tuple[Optional[str], str]]:
+        """Convert a reStructuredText file to a dictionary.
+
+        The keys are the headers and the values are the text under each header.
+
+        """
+        rst_tups: List[Tuple[Optional[str], str]] = []
+        lines = rst_text.split("\n")
+
+        current_header = None
+        current_text = ""
+
+        for i, line in enumerate(lines):
+            header_match = re.match(r"^[^\S\n]*[-=]+[^\S\n]*$", line)
+            if header_match and i > 0 and (len(lines[i - 1].strip()) == len(header_match.group().strip()) or lines[i - 2] == lines[i - 2]):
+                if current_header is not None:
+                    if current_text == "" or None:
+                        continue
+                    # removes the next heading from current Document
+                    if current_text.endswith(lines[i - 1] + "\n"):
+                        current_text = current_text[:len(current_text) - len(lines[i - 1] + "\n")]
+                    rst_tups.append((current_header, current_text))
+
+                current_header = lines[i - 1]
+                current_text = ""
+            else:
+                current_text += line + "\n"
+
+        rst_tups.append((current_header, current_text))
+
+        #TODO: Format for rst
+        #
+        # if current_header is not None:
+        #     # pass linting, assert keys are defined
+        #     rst_tups = [
+        #         (re.sub(r"#", "", cast(str, key)).strip(), re.sub(r"<.*?>", "", value))
+        #         for key, value in rst_tups
+        #     ]
+        # else:
+        #     rst_tups = [
+        #         (key, re.sub("\n", "", value)) for key, value in rst_tups
+        #     ]
+
+        if current_header is None:
+            rst_tups = [
+                (key, re.sub("\n", "", value)) for key, value in rst_tups
+            ]
+        return rst_tups
+
+    def remove_images(self, content: str) -> str:
+        pattern = r"\.\. image:: (.*)"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_hyperlinks(self, content: str) -> str:
+        pattern = r"`(.*?) <(.*?)>`_"
+        content = re.sub(pattern, r"\1", content)
+        return content
+
+    def remove_directives(self, content: str) -> str:
+        """Removes reStructuredText Directives"""
+        pattern = r"`\.\.([^:]+)::"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_interpreters(self, content: str) -> str:
+        """Removes reStructuredText Interpreted Text Roles"""
+        pattern = r":(\w+):"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_table_excess(self, content: str) -> str:
+        """Pattern to remove grid table separators"""
+        pattern = r"^\+[-]+\+[-]+\+$"
+        content = re.sub(pattern, "", content, flags=re.MULTILINE)
+        return content
+
+    def remove_whitespaces_excess(self, content: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:
+        """Pattern to match 2 or more consecutive whitespaces"""
+        pattern = r"\s{2,}"
+        content = [(key, re.sub(pattern, "  ", value)) for key, value in content]
+        return content
+
+    def remove_characters_excess(self, content: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:
+        """Pattern to match 2 or more consecutive characters"""
+        pattern = r"(\S)\1{2,}"
+        content = [(key, re.sub(pattern, r"\1\1\1", value, flags=re.MULTILINE)) for key, value in content]
+        return content
+
+    def _init_parser(self) -> Dict:
+        """Initialize the parser with the config."""
+        return {}
+
+    def parse_tups(
+        self, filepath: Path, errors: str = "ignore"
+    ) -> List[Tuple[Optional[str], str]]:
+        """Parse file into tuples."""
+        with open(filepath, "r") as f:
+            content = f.read()
+        if self._remove_hyperlinks:
+            content = self.remove_hyperlinks(content)
+        if self._remove_images:
+            content = self.remove_images(content)
+        if self._remove_table_excess:
+            content = self.remove_table_excess(content)
+        if self._remove_directives:
+            content = self.remove_directives(content)
+        if self._remove_interpreters:
+            content = self.remove_interpreters(content)
+        rst_tups = self.rst_to_tups(content)
+        if self._remove_whitespaces_excess:
+            rst_tups = self.remove_whitespaces_excess(rst_tups)
+        if self._remove_characters_excess:
+            rst_tups = self.remove_characters_excess(rst_tups)
+        return rst_tups
+
+    def parse_file(
+        self, filepath: Path, errors: str = "ignore"
+    ) -> Union[str, List[str]]:
+        """Parse file into string."""
+        tups = self.parse_tups(filepath, errors=errors)
+        results = []
+        # TODO: don't include headers right now
+        for header, value in tups:
+            if header is None:
+                results.append(value)
+            else:
+                results.append(f"\n\n{header}\n{value}")
+        return results

application/parser/file/tabular_parser.py

@@ -0,0 +1,115 @@
+"""Tabular parser.
+
+Contains parsers for tabular data files.
+
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from parser.file.base_parser import BaseParser
+
+
+class CSVParser(BaseParser):
+    """CSV parser.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+    """
+
+    def __init__(self, *args: Any, concat_rows: bool = True, **kwargs: Any) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file.
+
+        Returns:
+            Union[str, List[str]]: a string or a List of strings.
+
+        """
+        try:
+            import csv
+        except ImportError:
+            raise ValueError("csv module is required to read CSV files.")
+        text_list = []
+        with open(file, "r") as fp:
+            csv_reader = csv.reader(fp)
+            for row in csv_reader:
+                text_list.append(", ".join(row))
+        if self._concat_rows:
+            return "\n".join(text_list)
+        else:
+            return text_list
+
+
+class PandasCSVParser(BaseParser):
+    r"""Pandas-based CSV parser.
+
+    Parses CSVs using the separator detection from Pandas `read_csv`function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_csv` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the separators, table head, etc. on its own.
+
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        concat_rows: bool = True,
+        col_joiner: str = ", ",
+        row_joiner: str = "\n",
+        pandas_config: dict = {},
+        **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read CSV files.")
+
+        df = pd.read_csv(file, **self._pandas_config)
+
+        text_list = df.apply(
+            lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
+        ).tolist()
+
+        if self._concat_rows:
+            return (self._row_joiner).join(text_list)
+        else:
+            return text_list

application/parser/java2doc.py

@@ -0,0 +1,61 @@
+import os
+import javalang
+
+def find_files(directory):
+    files_list = []
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith('.java'):
+                files_list.append(os.path.join(root, file))
+    return files_list
+
+def extract_functions(file_path):
+    with open(file_path, "r") as file:
+        java_code = file.read()
+        methods = {}
+        tree = javalang.parse.parse(java_code)
+        for _, node in tree.filter(javalang.tree.MethodDeclaration):
+            method_name = node.name
+            start_line = node.position.line - 1
+            end_line = start_line
+            brace_count = 0
+            for line in java_code.splitlines()[start_line:]:
+                end_line += 1
+                brace_count += line.count("{") - line.count("}")
+                if brace_count == 0:
+                    break
+            method_source_code = "\n".join(java_code.splitlines()[start_line:end_line])
+            methods[method_name] = method_source_code
+    return methods
+
+def extract_classes(file_path):
+    with open(file_path, 'r') as file:
+        source_code = file.read()
+        classes = {}
+        tree = javalang.parse.parse(source_code)
+        for class_decl in tree.types:
+            class_name = class_decl.name
+            declarations = []
+            methods = []
+            for field_decl in class_decl.fields:
+                field_name = field_decl.declarators[0].name
+                field_type = field_decl.type.name
+                declarations.append(f"{field_type} {field_name}")
+            for method_decl in class_decl.methods:
+                methods.append(method_decl.name)
+            class_string = "Declarations: " + ", ".join(declarations) + "\n  Method name: " + ", ".join(methods)
+            classes[class_name] = class_string
+    return classes
+
+def extract_functions_and_classes(directory):
+    files = find_files(directory)
+    functions_dict = {}
+    classes_dict = {}
+    for file in files:
+        functions = extract_functions(file)
+        if functions:
+            functions_dict[file] = functions
+        classes = extract_classes(file)
+        if classes:
+            classes_dict[file] = classes
+    return functions_dict, classes_dict

application/parser/js2doc.py

@@ -0,0 +1,67 @@
+import os
+import esprima
+import escodegen
+
+
+def find_files(directory):
+    files_list = []
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith('.js'):
+                files_list.append(os.path.join(root, file))
+    return files_list
+
+def extract_functions(file_path):
+    with open(file_path, 'r') as file:
+        source_code = file.read()
+        functions = {}
+        tree = esprima.parseScript(source_code)
+        for node in tree.body:
+            if node.type == 'FunctionDeclaration':
+                func_name = node.id.name if node.id else '<anonymous>'
+                functions[func_name] = escodegen.generate(node)
+            elif node.type == 'VariableDeclaration':
+                for declaration in node.declarations:
+                    if declaration.init and declaration.init.type == 'FunctionExpression':
+                        func_name = declaration.id.name if declaration.id else '<anonymous>'
+                        functions[func_name] = escodegen.generate(declaration.init)
+            elif node.type == 'ClassDeclaration':
+                class_name = node.id.name
+                for subnode in node.body.body:
+                    if subnode.type == 'MethodDefinition':
+                        func_name = subnode.key.name
+                        functions[func_name] = escodegen.generate(subnode.value)
+                    elif subnode.type == 'VariableDeclaration':
+                        for declaration in subnode.declarations:
+                            if declaration.init and declaration.init.type == 'FunctionExpression':
+                                func_name = declaration.id.name if declaration.id else '<anonymous>'
+                                functions[func_name] = escodegen.generate(declaration.init)
+        return functions
+
+def extract_classes(file_path):
+    with open(file_path, 'r') as file:
+        source_code = file.read()
+        classes = {}
+        tree = esprima.parseScript(source_code)
+        for node in tree.body:
+            if node.type == 'ClassDeclaration':
+                class_name = node.id.name
+                function_names = []
+                for subnode in node.body.body:
+                    if subnode.type == 'MethodDefinition':
+                        function_names.append(subnode.key.name)
+                classes[class_name] = ", ".join(function_names)
+    return classes
+
+def extract_functions_and_classes(directory):
+    files = find_files(directory)
+    functions_dict = {}
+    classes_dict = {}
+    for file in files:
+        functions = extract_functions(file)
+        if functions:
+            functions_dict[file] = functions
+        classes = extract_classes(file)
+        if classes:
+            classes_dict[file] = classes
+    return functions_dict, classes_dict

application/parser/open_ai_func.py

@@ -0,0 +1,81 @@
+import os
+import faiss
+import pickle
+import tiktoken
+from langchain.vectorstores import FAISS
+from langchain.embeddings import OpenAIEmbeddings
+
+#from langchain.embeddings import HuggingFaceEmbeddings
+#from langchain.embeddings import HuggingFaceInstructEmbeddings
+#from langchain.embeddings import CohereEmbeddings
+
+from retry import retry
+
+
+
+def num_tokens_from_string(string: str, encoding_name: str) -> int:
+# Function to convert string to tokens and estimate user cost.
+    encoding = tiktoken.get_encoding(encoding_name)
+    num_tokens = len(encoding.encode(string))
+    total_price = ((num_tokens/1000) * 0.0004)
+    return num_tokens, total_price
+
+@retry(tries=10, delay=60)
+def store_add_texts_with_retry(store, i):
+    store.add_texts([i.page_content], metadatas=[i.metadata])
+    #store_pine.add_texts([i.page_content], metadatas=[i.metadata])
+
+def call_openai_api(docs, folder_name, task_status):
+# Function to create a vector store from the documents and save it to disk.
+
+    # create output folder if it doesn't exist
+    if not os.path.exists(f"{folder_name}"):
+        os.makedirs(f"{folder_name}")
+
+    from tqdm import tqdm
+    docs_test = [docs[0]]
+    docs.pop(0)
+    c1 = 0
+
+    store = FAISS.from_documents(docs_test, OpenAIEmbeddings(openai_api_key=os.getenv("EMBEDDINGS_KEY")))
+
+    # Uncomment for MPNet embeddings
+    # model_name = "sentence-transformers/all-mpnet-base-v2"
+    # hf = HuggingFaceEmbeddings(model_name=model_name)
+    # store = FAISS.from_documents(docs_test, hf)
+    s1 = len(docs)
+    for i in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs), bar_format='{l_bar}{bar}| Time Left: {remaining}'):
+        try:
+            task_status.update_state(state='PROGRESS', meta={'current': int((c1 / s1) * 100)})
+            store_add_texts_with_retry(store, i)
+        except Exception as e:
+            print(e)
+            print("Error on ", i)
+            print("Saving progress")
+            print(f"stopped at {c1} out of {len(docs)}")
+            store.save_local(f"{folder_name}")
+            break
+        c1 += 1
+    store.save_local(f"{folder_name}")
+
+def get_user_permission(docs, folder_name):
+# Function to ask user permission to call the OpenAI api and spend their OpenAI funds.
+    # Here we convert the docs list to a string and calculate the number of OpenAI tokens the string represents.
+    #docs_content = (" ".join(docs))
+    docs_content = ""
+    for doc in docs:
+        docs_content += doc.page_content
+
+
+    tokens, total_price = num_tokens_from_string(string=docs_content, encoding_name="cl100k_base")
+    # Here we print the number of tokens and the approx user cost with some visually appealing formatting.
+    print(f"Number of Tokens = {format(tokens, ',d')}")
+    print(f"Approx Cost = ${format(total_price, ',.2f')}")
+    #Here we check for user permission before calling the API.
+    user_input = input("Price Okay? (Y/N) \n").lower()
+    if user_input == "y":
+        call_openai_api(docs, folder_name)
+    elif user_input == "":
+        call_openai_api(docs, folder_name)
+    else:
+        print("The API was not called. No money was spent.")

application/parser/py2doc.py

@@ -0,0 +1,113 @@
+import os
+import ast
+import tiktoken
+from pathlib import Path
+from langchain.llms import OpenAI
+from langchain.prompts import PromptTemplate
+
+def find_files(directory):
+    files_list = []
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith('.py'):
+                files_list.append(os.path.join(root, file))
+    return files_list
+
+def extract_functions(file_path):
+    with open(file_path, 'r') as file:
+        source_code = file.read()
+        functions = {}
+        tree = ast.parse(source_code)
+        for node in ast.walk(tree):
+            if isinstance(node, ast.FunctionDef):
+                func_name = node.name
+                func_def = ast.get_source_segment(source_code, node)
+                functions[func_name] = func_def
+    return functions
+
+def extract_classes(file_path):
+    with open(file_path, 'r') as file:
+        source_code = file.read()
+        classes = {}
+        tree = ast.parse(source_code)
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ClassDef):
+                class_name = node.name
+                function_names = []
+                for subnode in ast.walk(node):
+                    if isinstance(subnode, ast.FunctionDef):
+                        function_names.append(subnode.name)
+                classes[class_name] = ", ".join(function_names)
+    return classes
+
+def extract_functions_and_classes(directory):
+    files = find_files(directory)
+    functions_dict = {}
+    classes_dict = {}
+    for file in files:
+        functions = extract_functions(file)
+        if functions:
+            functions_dict[file] = functions
+        classes = extract_classes(file)
+        if classes:
+            classes_dict[file] = classes
+    return functions_dict, classes_dict
+
+def parse_functions(functions_dict, formats, dir):
+    c1 = len(functions_dict)
+    for i, (source, functions) in enumerate(functions_dict.items(), start=1):
+        print(f"Processing file {i}/{c1}")
+        source_w = source.replace(dir+"/", "").replace("."+formats, ".md")
+        subfolders = "/".join(source_w.split("/")[:-1])
+        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
+        for j, (name, function) in enumerate(functions.items(), start=1):
+            print(f"Processing function {j}/{len(functions)}")
+            prompt = PromptTemplate(
+                input_variables=["code"],
+                template="Code: \n{code}, \nDocumentation: ",
+            )
+            llm = OpenAI(temperature=0)
+            response = llm(prompt.format(code=function))
+            mode = "a" if Path(f"outputs/{source_w}").exists() else "w"
+            with open(f"outputs/{source_w}", mode) as f:
+                f.write(f"\n\n# Function name: {name} \n\nFunction: \n```\n{function}\n```, \nDocumentation: \n{response}")
+
+
+def parse_classes(classes_dict, formats, dir):
+    c1 = len(classes_dict)
+    for i, (source, classes) in enumerate(classes_dict.items()):
+        print(f"Processing file {i+1}/{c1}")
+        source_w = source.replace(dir+"/", "").replace("."+formats, ".md")
+        subfolders = "/".join(source_w.split("/")[:-1])
+        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
+        for name, function_names in classes.items():
+            print(f"Processing Class {i+1}/{c1}")
+            prompt = PromptTemplate(
+                input_variables=["class_name", "functions_names"],
+                template="Class name: {class_name} \nFunctions: {functions_names}, \nDocumentation: ",
+            )
+            llm = OpenAI(temperature=0)
+            response = llm(prompt.format(class_name=name, functions_names=function_names))
+
+            with open(f"outputs/{source_w}", "a" if Path(f"outputs/{source_w}").exists() else "w") as f:
+                f.write(f"\n\n# Class name: {name} \n\nFunctions: \n{function_names}, \nDocumentation: \n{response}")
+
+def transform_to_docs(functions_dict, classes_dict, formats, dir):
+    docs_content = ''.join([str(key) + str(value) for key, value in functions_dict.items()])
+    docs_content += ''.join([str(key) + str(value) for key, value in classes_dict.items()])
+
+    num_tokens = len(tiktoken.get_encoding("cl100k_base").encode(docs_content))
+    total_price = ((num_tokens / 1000) * 0.02)
+
+    print(f"Number of Tokens = {num_tokens:,d}")
+    print(f"Approx Cost = ${total_price:,.2f}")
+
+    user_input = input("Price Okay? (Y/N)\n").lower()
+    if user_input == "y" or user_input == "":
+        if not Path("outputs").exists():
+            Path("outputs").mkdir()
+        parse_functions(functions_dict, formats, dir)
+        parse_classes(classes_dict, formats, dir)
+        print("All done!")
+    else:
+        print("The API was not called. No money was spent.")

application/parser/schema/base.py

@@ -0,0 +1,35 @@
+"""Base schema for readers."""
+from dataclasses import dataclass
+
+from langchain.docstore.document import Document as LCDocument
+
+from parser.schema.schema import BaseDocument
+
+
+@dataclass
+class Document(BaseDocument):
+    """Generic interface for a data document.
+
+    This document connects to data sources.
+
+    """
+
+    def __post_init__(self) -> None:
+        """Post init."""
+        if self.text is None:
+            raise ValueError("text field not set.")
+
+    @classmethod
+    def get_type(cls) -> str:
+        """Get Document type."""
+        return "Document"
+
+    def to_langchain_format(self) -> LCDocument:
+        """Convert struct to LangChain document format."""
+        metadata = self.extra_info or {}
+        return LCDocument(page_content=self.text, metadata=metadata)
+
+    @classmethod
+    def from_langchain_format(cls, doc: LCDocument) -> "Document":
+        """Convert struct from LangChain document format."""
+        return cls(text=doc.page_content, extra_info=doc.metadata)

application/parser/schema/schema.py

@@ -0,0 +1,64 @@
+"""Base schema for data structures."""
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from dataclasses_json import DataClassJsonMixin
+
+
+@dataclass
+class BaseDocument(DataClassJsonMixin):
+    """Base document.
+
+    Generic abstract interfaces that captures both index structs
+    as well as documents.
+
+    """
+
+    # TODO: consolidate fields from Document/IndexStruct into base class
+    text: Optional[str] = None
+    doc_id: Optional[str] = None
+    embedding: Optional[List[float]] = None
+
+    # extra fields
+    extra_info: Optional[Dict[str, Any]] = None
+
+    @classmethod
+    @abstractmethod
+    def get_type(cls) -> str:
+        """Get Document type."""
+
+    def get_text(self) -> str:
+        """Get text."""
+        if self.text is None:
+            raise ValueError("text field not set.")
+        return self.text
+
+    def get_doc_id(self) -> str:
+        """Get doc_id."""
+        if self.doc_id is None:
+            raise ValueError("doc_id not set.")
+        return self.doc_id
+
+    @property
+    def is_doc_id_none(self) -> bool:
+        """Check if doc_id is None."""
+        return self.doc_id is None
+
+    def get_embedding(self) -> List[float]:
+        """Get embedding.
+
+        Errors if embedding is None.
+
+        """
+        if self.embedding is None:
+            raise ValueError("embedding not set.")
+        return self.embedding
+
+    @property
+    def extra_info_str(self) -> Optional[str]:
+        """Extra info string."""
+        if self.extra_info is None:
+            return None
+
+        return "\n".join([f"{k}: {str(v)}" for k, v in self.extra_info.items()])

application/parser/token_func.py

@@ -0,0 +1,70 @@
+import re
+import tiktoken
+
+from typing import List
+from parser.schema.base import Document
+from math import ceil
+
+
+def separate_header_and_body(text):
+    header_pattern = r"^(.*?\n){3}"
+    match = re.match(header_pattern, text)
+    header = match.group(0)
+    body = text[len(header):]
+    return header, body
+
+def group_documents(documents: List[Document], min_tokens: int, max_tokens: int) -> List[Document]:
+    docs = []
+    current_group = None
+
+    for doc in documents:
+        doc_len = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
+
+        if current_group is None:
+            current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
+                                     extra_info=doc.extra_info)
+        elif len(tiktoken.get_encoding("cl100k_base").encode(current_group.text)) + doc_len < max_tokens and doc_len >= min_tokens:
+            current_group.text += " " + doc.text
+        else:
+            docs.append(current_group)
+            current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
+                                     extra_info=doc.extra_info)
+
+    if current_group is not None:
+        docs.append(current_group)
+
+    return docs
+
+def split_documents(documents: List[Document], max_tokens: int) -> List[Document]:
+    docs = []
+    for doc in documents:
+        token_length = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
+        if token_length <= max_tokens:
+            docs.append(doc)
+        else:
+            header, body = separate_header_and_body(doc.text)
+            num_body_parts = ceil(token_length / max_tokens)
+            part_length = ceil(len(body) / num_body_parts)
+            body_parts = [body[i:i + part_length] for i in range(0, len(body), part_length)]
+            for i, body_part in enumerate(body_parts):
+                new_doc = Document(text=header + body_part.strip(),
+                                   doc_id=f"{doc.doc_id}-{i}",
+                                   embedding=doc.embedding,
+                                   extra_info=doc.extra_info)
+                docs.append(new_doc)
+    return docs
+
+def group_split(documents: List[Document], max_tokens: int = 2000, min_tokens: int = 150, token_check: bool = True):
+    if token_check == False:
+        return documents
+    print("Grouping small documents")
+    try:
+        documents = group_documents(documents=documents, min_tokens=min_tokens, max_tokens=max_tokens)
+    except:
+        print("Grouping failed, try running without token_check")
+    print("Separating large documents")
+    try:
+        documents = split_documents(documents=documents, max_tokens=max_tokens)
+    except:
+        print("Grouping failed, try running without token_check")
+    return documents

application/prompts/chat_combine_prompt.txt

@@ -0,0 +1,4 @@
+You are a DocsGPT, friendly and helpful AI assistant by Arc53 that provides help with documents. You give thorough answers with code examples if possible.
+Use the following pieces of context to help answer the users question.
+----------------
+{summaries}

application/prompts/chat_reduce_prompt.txt

@@ -0,0 +1,3 @@
+Use the following portion of a long document to see if any of the text is relevant to answer the question.
+{context}
+Provide all relevant text to the question verbatim. Summarize if needed. If nothing relevant return "-".

application/requirements.txt

@@ -1,126 +1,92 @@
 aiodns==3.0.0
-aiohttp==3.8.3
+aiohttp==3.8.4
 aiohttp-retry==2.8.3
 aiosignal==1.3.1
-alabaster==0.7.13
-aleph-alpha-client==2.16.0
-anyio==3.6.2
-argilla==1.3.0
+aleph-alpha-client==2.16.1
+amqp==5.1.1
 async-timeout==4.0.2
 attrs==22.2.0
-Babel==2.11.0
-backoff==2.2.1
+billiard==3.6.4.0
 blobfile==2.0.1
-boto3==1.26.82
-botocore==1.29.82
+boto3==1.26.84
+botocore==1.29.84
 cffi==1.15.1
-charset-normalizer==2.1.1
+charset-normalizer==3.1.0
 click==8.1.3
-cohere==3.4.0
+click-didyoumean==0.3.0
+click-plugins==1.1.1
+click-repl==0.2.0
+cryptography==39.0.2
 dataclasses-json==0.5.7
 decorator==5.1.1
-deeplake==3.2.12
-Deprecated==1.2.13
+deeplake==3.2.13
 dill==0.3.6
-docutils==0.19
-docx2txt==0.8
+dnspython==2.3.0
+ecdsa==0.18.0
 entrypoints==0.4
-escodegen==1.0.10
-esprima==4.0.1
-esutils==1.0.1
-et-xmlfile==1.1.0
 faiss-cpu==1.7.3
 filelock==3.9.0
-Flask==2.2.2
+Flask==2.2.3
 frozenlist==1.3.3
 greenlet==2.0.2
-gunicorn==20.1.0
-h11==0.14.0
-httpcore==0.16.3
-httpx==0.23.3
 hub==3.0.1
-huggingface-hub==0.12.0
+huggingface-hub==0.12.1
 humbug==0.2.8
 idna==3.4
-imagesize==1.4.1
 itsdangerous==2.1.2
-javalang==0.13.0
 Jinja2==3.1.2
 jmespath==1.0.1
 joblib==1.2.0
-langchain==0.0.98
+kombu==5.2.4
+langchain==0.0.118
 lxml==4.9.2
-manifest-ml==0.1.1
 MarkupSafe==2.1.2
 marshmallow==3.19.0
 marshmallow-enum==1.5.1
-monotonic==1.6
 multidict==6.0.4
 multiprocess==0.70.14
-mypy-extensions==0.4.3
+mypy-extensions==1.0.0
 nltk==3.8.1
 numcodecs==0.11.0
-numpy==1.23.5
+numpy==1.24.2
 openai==0.27.0
-openpyxl==3.1.1
 packaging==23.0
-pandas==1.5.3
 pathos==0.3.0
 Pillow==9.4.0
 pox==0.3.2
 ppft==1.7.6.6
+prompt-toolkit==3.0.38
 py==1.11.0
+pyasn1==0.4.8
 pycares==4.3.0
 pycparser==2.21
 pycryptodomex==3.17
-pydantic==1.10.4
-Pygments==2.14.0
+pydantic==1.10.5
 PyJWT==2.6.0
+pymongo==4.3.3
 PyPDF2==3.0.1
 python-dateutil==2.8.2
-python-docx==0.8.11
-python-dotenv==0.21.1
-python-magic==0.4.27
-python-pptx==0.6.21
+python-dotenv==1.0.0
+python-jose==3.3.0
 pytz==2022.7.1
 PyYAML==6.0
-redis==4.5.1
+redis==4.5.2
 regex==2022.10.31
 requests==2.28.2
 retry==0.9.2
-rfc3986==1.5.0
+rsa==4.9
 s3transfer==0.6.0
-scikit-learn==1.2.1
-scipy==1.10.0
-sentence-transformers==2.2.2
-sentencepiece==0.1.97
 six==1.16.0
-sniffio==1.3.0
-snowballstemmer==2.2.0
-Sphinx==6.1.3
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
 SQLAlchemy==1.4.46
-sqlitedict==2.1.0
-tenacity==8.2.1
-threadpoolctl==3.1.0
-tiktoken==0.1.2
+tenacity==8.2.2
+tiktoken==0.3.0
 tokenizers==0.13.2
-torch==1.13.1
-torchvision==0.14.1
-tqdm==4.64.1
-transformers==4.26.0
-typer==0.7.0
+tqdm==4.65.0
+transformers==4.26.1
 typing-inspect==0.8.0
-typing_extensions==4.4.0
-unstructured==0.4.11
+typing_extensions==4.5.0
 urllib3==1.26.14
+vine==5.0.0
+wcwidth==0.2.6
 Werkzeug==2.2.3
-wrapt==1.14.1
-XlsxWriter==3.0.8
-xxhash==3.2.0
 yarl==1.8.2

application/static/dist/css/output.css

@@ -525,6 +525,10 @@ video {
   position: absolute;
 }
 
+.relative {
+  position: relative;
+}
+
 .inset-0 {
   top: 0px;
   right: 0px;
@@ -604,6 +608,10 @@ video {
   min-height: 100vh;
 }
 
+.w-auto {
+  width: auto;
+}
+
 .w-full {
   width: 100%;
 }
@@ -648,12 +656,16 @@ video {
   overflow-y: auto;
 }
 
+.rounded {
+  border-radius: 0.25rem;
+}
+
 .rounded-lg {
   border-radius: 0.5rem;
 }
 
-.rounded {
-  border-radius: 0.25rem;
+.rounded-md {
+  border-radius: 0.375rem;
 }
 
 .border {
@@ -723,6 +735,11 @@ video {
   padding-bottom: 0.5rem;
 }
 
+.py-4 {
+  padding-top: 1rem;
+  padding-bottom: 1rem;
+}
+
 .pt-4 {
   padding-top: 1rem;
 }
@@ -761,6 +778,11 @@ video {
   line-height: 1.25rem;
 }
 
+.text-xl {
+  font-size: 1.25rem;
+  line-height: 1.75rem;
+}
+
 .font-medium {
   font-weight: 500;
 }
@@ -842,6 +864,11 @@ video {
   }
 }
 
+.hover\:bg-blue-600:hover {
+  --tw-bg-opacity: 1;
+  background-color: rgb(37 99 235 / var(--tw-bg-opacity));
+}
+
 .hover\:bg-blue-700:hover {
   --tw-bg-opacity: 1;
   background-color: rgb(29 78 216 / var(--tw-bg-opacity));
@@ -862,11 +889,26 @@ video {
   border-color: rgb(59 130 246 / var(--tw-border-opacity));
 }
 
+.focus\:outline-none:focus {
+  outline: 2px solid transparent;
+  outline-offset: 2px;
+}
+
+.focus\:ring-2:focus {
+  --tw-ring-offset-shadow: var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);
+  --tw-ring-shadow: var(--tw-ring-inset) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color);
+  box-shadow: var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow, 0 0 #0000);
+}
+
 .focus\:ring-blue-500:focus {
   --tw-ring-opacity: 1;
   --tw-ring-color: rgb(59 130 246 / var(--tw-ring-opacity));
 }
 
+.focus\:ring-offset-2:focus {
+  --tw-ring-offset-width: 2px;
+}
+
 @media (min-width: 640px) {
   .sm\:my-8 {
     margin-top: 2rem;
@@ -881,6 +923,10 @@ video {
     display: inline-block;
   }
 
+  .sm\:inline {
+    display: inline;
+  }
+
   .sm\:h-screen {
     height: 100vh;
   }

application/static/src/chat.js

@@ -71,4 +71,6 @@ function submitForm(event){
         });
 }
 
-window.addEventListener('submit',submitForm)
+//window.addEventListener('submit',submitForm)
+// rewrite using id = button-submit
+document.getElementById("button-submit").addEventListener('click',submitForm)

application/templates/index.html

@@ -86,6 +86,19 @@ This will return a new DataFrame with all the columns from both tables, and only
               <option selected value="default">Choose documentation</option>
               <option value="default">Default</option>
             </select>
+            <form action="/api/upload" method="post" enctype="multipart/form-data" class="mt-2">
+                <input type="file" name="file" class="py-4" id="file-upload">
+                <input type="text" name="user" value="local" hidden>
+                <input type="text" name="name" placeholder="Name:">
+
+
+              <button type="submit" class="py-2 px-4 text-white bg-blue-500 rounded-md hover:bg-blue-600 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500">
+                Upload
+              </button>
+            </form>
+
+
+
         </div>
     </div>
 
@@ -130,7 +143,7 @@ This will return a new DataFrame with all the columns from both tables, and only
           function docsIndex() {
                 // loads latest index from https://raw.githubusercontent.com/arc53/DocsHUB/main/combined.json
                 // and stores it in localStorage
-                fetch('https://d3dg1063dc54p9.cloudfront.net/combined.json')
+                fetch('/api/combine')
                     .then(response => response.json())
                     .then(data => {
                         localStorage.setItem("docsIndex", JSON.stringify(data));
@@ -150,19 +163,26 @@ This will return a new DataFrame with all the columns from both tables, and only
                 // create option for each key in docsIndex
                 for (var key in docsIndex) {
                     var option = document.createElement("option");
-                    if (docsIndex[key].name == docsIndex[key].language) {
-                        option.text = docsIndex[key].name + " " + docsIndex[key].version;
-                        option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
-                        if (docsIndex[key].model == "{{ embeddings_choice }}") {
-                            select.add(option);
+                    if (docsIndex[key].location == 'docshub'){
+                        if (docsIndex[key].name == docsIndex[key].language) {
+                            option.text = docsIndex[key].name + " " + docsIndex[key].version;
+                            option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
+                            if (docsIndex[key].model == "{{ embeddings_choice }}") {
+                                select.add(option);
+                            }
+                        }
+                        else {
+                            option.text = docsIndex[key].name + " " + docsIndex[key].version;
+                            option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
+                            if (docsIndex[key].model == "{{ embeddings_choice }}") {
+                                select.add(option);
+                            }
                         }
                     }
                     else {
-                        option.text = docsIndex[key].name + " " + docsIndex[key].version;
-                        option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
-                        if (docsIndex[key].model == "{{ embeddings_choice }}") {
-                            select.add(option);
-                        }
+                        option.text = docsIndex[key].name;
+                        option.value = docsIndex[key].location + "/" + docsIndex[key].name;
+                        select.add(option);
                     }
                 }
 

application/worker.py

@@ -0,0 +1,99 @@
+import requests
+import nltk
+import os
+
+from parser.file.bulk import SimpleDirectoryReader
+from parser.schema.base import Document
+from parser.open_ai_func import call_openai_api
+from parser.token_func import group_split
+from celery import current_task
+
+
+import string
+import zipfile
+import shutil
+
+try:
+    nltk.download('punkt', quiet=True)
+    nltk.download('averaged_perceptron_tagger', quiet=True)
+except FileExistsError:
+    pass
+def generate_random_string(length):
+    return ''.join([string.ascii_letters[i % 52] for i in range(length)])
+
+
+
+def ingest_worker(self, directory, formats, name_job, filename, user):
+    # directory = 'inputs' or 'temp'
+    # formats = [".rst", ".md"]
+    input_files = None
+    recursive = True
+    limit = None
+    exclude = True
+    # name_job = 'job1'
+    # filename = 'install.rst'
+    # user = 'local'
+    sample = False
+    token_check = True
+    min_tokens = 150
+    max_tokens = 1250
+    full_path = directory + '/' + user + '/' + name_job
+    # check if API_URL env variable is set
+    if not os.environ.get('API_URL'):
+        url = 'http://localhost:5001/api/download'
+    else:
+        url = os.environ.get('API_URL') + '/api/download'
+    file_data = {'name': name_job, 'file': filename, 'user': user}
+    response = requests.get(url, params=file_data)
+    file = response.content
+
+    if not os.path.exists(full_path):
+        os.makedirs(full_path)
+    with open(full_path + '/' + filename, 'wb') as f:
+        f.write(file)
+
+    #check if file is .zip and extract it
+    if filename.endswith('.zip'):
+        with zipfile.ZipFile(full_path + '/' + filename, 'r') as zip_ref:
+            zip_ref.extractall(full_path)
+        os.remove(full_path + '/' + filename)
+
+
+    import time
+    self.update_state(state='PROGRESS', meta={'current': 1})
+
+    raw_docs = SimpleDirectoryReader(input_dir=full_path, input_files=input_files, recursive=recursive,
+                                     required_exts=formats, num_files_limit=limit,
+                                     exclude_hidden=exclude).load_data()
+    raw_docs = group_split(documents=raw_docs, min_tokens=min_tokens, max_tokens=max_tokens, token_check=token_check)
+
+    docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+
+    call_openai_api(docs, full_path, self)
+    self.update_state(state='PROGRESS', meta={'current': 100})
+
+    if sample == True:
+        for i in range(min(5, len(raw_docs))):
+            print(raw_docs[i].text)
+
+    # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
+    # and send them to the server (provide user and name in form)
+    if not os.environ.get('API_URL'):
+        url = 'http://localhost:5001/api/upload_index'
+    else:
+        url = os.environ.get('API_URL') + '/api/upload_index'
+    file_data = {'name': name_job, 'user': user}
+    files = {'file_faiss': open(full_path + '/index.faiss', 'rb'),
+             'file_pkl': open(full_path + '/index.pkl', 'rb')}
+    response = requests.post(url, files=files, data=file_data)
+
+    #deletes remote
+    if not os.environ.get('API_URL'):
+        url = 'http://localhost:5001/api/delete_old?path=' + 'inputs/' + user + '/' + name_job
+    else:
+        url = os.environ.get('API_URL') + '/api/delete_old?path=' + 'inputs/' + user + '/' + name_job
+    response = requests.get(url)
+    # delete local
+    shutil.rmtree(full_path)
+
+    return {'directory': directory, 'formats': formats, 'name_job': name_job, 'filename': filename, 'user': user}

docker-compose.yaml

@@ -4,12 +4,56 @@ services:
   frontend:
     build: ./frontend
     environment:
-      - API_HOST=http://backend:5001
+      - VITE_API_HOST=http://localhost:5001
     ports:
       - "5173:5173"
+    depends_on:
+        - backend
 
   backend:
     build: ./application
+    environment:
+      - API_KEY=<your_api_key>
+      - EMBEDDINGS_KEY=<your_api_key>
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/1
+      - MONGO_URI=mongodb://mongo:27017/docsgpt
     ports:
       - "5001:5001"
+    volumes:
+      - app_data_container:/app
+    depends_on:
+        - redis
+        - mongo
 
+  worker:
+    build: ./application
+    command: celery -A app.celery worker -l INFO
+    environment:
+      - API_KEY=<your_api_key>
+      - EMBEDDINGS_KEY=<your_api_key>
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/1
+      - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - API_URL=http://backend:5001
+    depends_on:
+        - redis
+        - mongo
+
+  redis:
+    image: redis:6-alpine
+    ports:
+      - 6379:6379
+
+  mongo:
+    image: mongo:6
+    ports:
+      - 27017:27017
+    volumes:
+      - mongodb_data_container:/data/db
+
+
+
+volumes:
+  mongodb_data_container:
+  app_data_container:

extensions/chatwoot/.env_sample

@@ -0,0 +1,6 @@
+docsgpt_url=<docsgpt_api_url>
+chatwoot_url=<chatwoot_url>
+docsgpt_key=<openai_api_key or other llm>
+chatwoot_token=xxxxx
+account_id=(optional) 1
+assignee_id=(optional) 1

extensions/chatwoot/app.py

@@ -2,13 +2,16 @@ import requests
 import dotenv
 import os
 import json
+import pprint
 
 dotenv.load_dotenv()
 docsgpt_url = os.getenv("docsgpt_url")
 chatwoot_url = os.getenv("chatwoot_url")
 docsgpt_key = os.getenv("docsgpt_key")
 chatwoot_token = os.getenv("chatwoot_token")
-
+#account_id = os.getenv("account_id")
+#assignee_id = os.getenv("assignee_id")
+label_stop = "human-requested"
 
 def send_to_bot(sender, message):
     data = {
@@ -47,31 +50,38 @@ app = Flask(__name__)
 @app.route('/docsgpt', methods=['POST'])
 def docsgpt():
     data = request.get_json()
-    message_type = data['message_type']
+    pp = pprint.PrettyPrinter(indent=4)
+    pp.pprint(data)
+    try:
+        message_type = data['message_type']
+    except KeyError:
+        return "Not a message"
     message = data['content']
     conversation = data['conversation']['id']
     contact = data['sender']['id']
     account = data['account']['id']
+    assignee = data['conversation']['meta']['assignee']['id']
+    print(account)
+    print(label_stop)
+    print(data['conversation']['labels'])
+    print(assignee)
+
+    if label_stop in data['conversation']['labels']:
+        return "Label stop"
+    # elif str(account) != str(account_id):
+    #     return "Not the right account"
+
+    # elif str(assignee) != str(assignee_id):
+    #     return "Not the right assignee"
 
     if(message_type == "incoming"):
         bot_response = send_to_bot(contact, message)
         create_message = send_to_chatwoot(
             account, conversation, bot_response)
-        response = requests.post(
-            url="https://86x89umx77.execute-api.eu-west-2.amazonaws.com/docsgpt-logs",
-            headers={
-                "Content-Type": "application/json; charset=utf-8",
-            },
-            data=json.dumps({
-                "answer": str(bot_response),
-                "question": str(message),
-                "source": "chatwoot"
-            })
-        )
     else:
         return "Not an incoming message"
 
     return create_message
 
 if __name__ == '__main__':
-    app.run(debug=True, host='0.0.0.0')
+    app.run(host='0.0.0.0', port=80)

extensions/chrome/dist/output.css

@@ -0,0 +1,678 @@
+/*
+! tailwindcss v3.2.7 | MIT License | https://tailwindcss.com
+*/
+
+/*
+1. Prevent padding and border from affecting element width. (https://github.com/mozdevs/cssremedy/issues/4)
+2. Allow adding a border to an element by just adding a border-width. (https://github.com/tailwindcss/tailwindcss/pull/116)
+*/
+
+*,
+::before,
+::after {
+  box-sizing: border-box;
+  /* 1 */
+  border-width: 0;
+  /* 2 */
+  border-style: solid;
+  /* 2 */
+  border-color: #e5e7eb;
+  /* 2 */
+}
+
+::before,
+::after {
+  --tw-content: '';
+}
+
+/*
+1. Use a consistent sensible line-height in all browsers.
+2. Prevent adjustments of font size after orientation changes in iOS.
+3. Use a more readable tab size.
+4. Use the user's configured `sans` font-family by default.
+5. Use the user's configured `sans` font-feature-settings by default.
+*/
+
+html {
+  line-height: 1.5;
+  /* 1 */
+  -webkit-text-size-adjust: 100%;
+  /* 2 */
+  -moz-tab-size: 4;
+  /* 3 */
+  -o-tab-size: 4;
+     tab-size: 4;
+  /* 3 */
+  font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+  /* 4 */
+  font-feature-settings: normal;
+  /* 5 */
+}
+
+/*
+1. Remove the margin in all browsers.
+2. Inherit line-height from `html` so users can set them as a class directly on the `html` element.
+*/
+
+body {
+  margin: 0;
+  /* 1 */
+  line-height: inherit;
+  /* 2 */
+}
+
+/*
+1. Add the correct height in Firefox.
+2. Correct the inheritance of border color in Firefox. (https://bugzilla.mozilla.org/show_bug.cgi?id=190655)
+3. Ensure horizontal rules are visible by default.
+*/
+
+hr {
+  height: 0;
+  /* 1 */
+  color: inherit;
+  /* 2 */
+  border-top-width: 1px;
+  /* 3 */
+}
+
+/*
+Add the correct text decoration in Chrome, Edge, and Safari.
+*/
+
+abbr:where([title]) {
+  -webkit-text-decoration: underline dotted;
+          text-decoration: underline dotted;
+}
+
+/*
+Remove the default font size and weight for headings.
+*/
+
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  font-size: inherit;
+  font-weight: inherit;
+}
+
+/*
+Reset links to optimize for opt-in styling instead of opt-out.
+*/
+
+a {
+  color: inherit;
+  text-decoration: inherit;
+}
+
+/*
+Add the correct font weight in Edge and Safari.
+*/
+
+b,
+strong {
+  font-weight: bolder;
+}
+
+/*
+1. Use the user's configured `mono` font family by default.
+2. Correct the odd `em` font sizing in all browsers.
+*/
+
+code,
+kbd,
+samp,
+pre {
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+  /* 1 */
+  font-size: 1em;
+  /* 2 */
+}
+
+/*
+Add the correct font size in all browsers.
+*/
+
+small {
+  font-size: 80%;
+}
+
+/*
+Prevent `sub` and `sup` elements from affecting the line height in all browsers.
+*/
+
+sub,
+sup {
+  font-size: 75%;
+  line-height: 0;
+  position: relative;
+  vertical-align: baseline;
+}
+
+sub {
+  bottom: -0.25em;
+}
+
+sup {
+  top: -0.5em;
+}
+
+/*
+1. Remove text indentation from table contents in Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=999088, https://bugs.webkit.org/show_bug.cgi?id=201297)
+2. Correct table border color inheritance in all Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=935729, https://bugs.webkit.org/show_bug.cgi?id=195016)
+3. Remove gaps between table borders by default.
+*/
+
+table {
+  text-indent: 0;
+  /* 1 */
+  border-color: inherit;
+  /* 2 */
+  border-collapse: collapse;
+  /* 3 */
+}
+
+/*
+1. Change the font styles in all browsers.
+2. Remove the margin in Firefox and Safari.
+3. Remove default padding in all browsers.
+*/
+
+button,
+input,
+optgroup,
+select,
+textarea {
+  font-family: inherit;
+  /* 1 */
+  font-size: 100%;
+  /* 1 */
+  font-weight: inherit;
+  /* 1 */
+  line-height: inherit;
+  /* 1 */
+  color: inherit;
+  /* 1 */
+  margin: 0;
+  /* 2 */
+  padding: 0;
+  /* 3 */
+}
+
+/*
+Remove the inheritance of text transform in Edge and Firefox.
+*/
+
+button,
+select {
+  text-transform: none;
+}
+
+/*
+1. Correct the inability to style clickable types in iOS and Safari.
+2. Remove default button styles.
+*/
+
+button,
+[type='button'],
+[type='reset'],
+[type='submit'] {
+  -webkit-appearance: button;
+  /* 1 */
+  background-color: transparent;
+  /* 2 */
+  background-image: none;
+  /* 2 */
+}
+
+/*
+Use the modern Firefox focus style for all focusable elements.
+*/
+
+:-moz-focusring {
+  outline: auto;
+}
+
+/*
+Remove the additional `:invalid` styles in Firefox. (https://github.com/mozilla/gecko-dev/blob/2f9eacd9d3d995c937b4251a5557d95d494c9be1/layout/style/res/forms.css#L728-L737)
+*/
+
+:-moz-ui-invalid {
+  box-shadow: none;
+}
+
+/*
+Add the correct vertical alignment in Chrome and Firefox.
+*/
+
+progress {
+  vertical-align: baseline;
+}
+
+/*
+Correct the cursor style of increment and decrement buttons in Safari.
+*/
+
+::-webkit-inner-spin-button,
+::-webkit-outer-spin-button {
+  height: auto;
+}
+
+/*
+1. Correct the odd appearance in Chrome and Safari.
+2. Correct the outline style in Safari.
+*/
+
+[type='search'] {
+  -webkit-appearance: textfield;
+  /* 1 */
+  outline-offset: -2px;
+  /* 2 */
+}
+
+/*
+Remove the inner padding in Chrome and Safari on macOS.
+*/
+
+::-webkit-search-decoration {
+  -webkit-appearance: none;
+}
+
+/*
+1. Correct the inability to style clickable types in iOS and Safari.
+2. Change font properties to `inherit` in Safari.
+*/
+
+::-webkit-file-upload-button {
+  -webkit-appearance: button;
+  /* 1 */
+  font: inherit;
+  /* 2 */
+}
+
+/*
+Add the correct display in Chrome and Safari.
+*/
+
+summary {
+  display: list-item;
+}
+
+/*
+Removes the default spacing and border for appropriate elements.
+*/
+
+blockquote,
+dl,
+dd,
+h1,
+h2,
+h3,
+h4,
+h5,
+h6,
+hr,
+figure,
+p,
+pre {
+  margin: 0;
+}
+
+fieldset {
+  margin: 0;
+  padding: 0;
+}
+
+legend {
+  padding: 0;
+}
+
+ol,
+ul,
+menu {
+  list-style: none;
+  margin: 0;
+  padding: 0;
+}
+
+/*
+Prevent resizing textareas horizontally by default.
+*/
+
+textarea {
+  resize: vertical;
+}
+
+/*
+1. Reset the default placeholder opacity in Firefox. (https://github.com/tailwindlabs/tailwindcss/issues/3300)
+2. Set the default placeholder color to the user's configured gray 400 color.
+*/
+
+input::-moz-placeholder, textarea::-moz-placeholder {
+  opacity: 1;
+  /* 1 */
+  color: #9ca3af;
+  /* 2 */
+}
+
+input::placeholder,
+textarea::placeholder {
+  opacity: 1;
+  /* 1 */
+  color: #9ca3af;
+  /* 2 */
+}
+
+/*
+Set the default cursor for buttons.
+*/
+
+button,
+[role="button"] {
+  cursor: pointer;
+}
+
+/*
+Make sure disabled buttons don't get the pointer cursor.
+*/
+
+:disabled {
+  cursor: default;
+}
+
+/*
+1. Make replaced elements `display: block` by default. (https://github.com/mozdevs/cssremedy/issues/14)
+2. Add `vertical-align: middle` to align replaced elements more sensibly by default. (https://github.com/jensimmons/cssremedy/issues/14#issuecomment-634934210)
+   This can trigger a poorly considered lint error in some tools but is included by design.
+*/
+
+img,
+svg,
+video,
+canvas,
+audio,
+iframe,
+embed,
+object {
+  display: block;
+  /* 1 */
+  vertical-align: middle;
+  /* 2 */
+}
+
+/*
+Constrain images and videos to the parent width and preserve their intrinsic aspect ratio. (https://github.com/mozdevs/cssremedy/issues/14)
+*/
+
+img,
+video {
+  max-width: 100%;
+  height: auto;
+}
+
+/* Make elements with the HTML hidden attribute stay hidden by default */
+
+[hidden] {
+  display: none;
+}
+
+*, ::before, ::after {
+  --tw-border-spacing-x: 0;
+  --tw-border-spacing-y: 0;
+  --tw-translate-x: 0;
+  --tw-translate-y: 0;
+  --tw-rotate: 0;
+  --tw-skew-x: 0;
+  --tw-skew-y: 0;
+  --tw-scale-x: 1;
+  --tw-scale-y: 1;
+  --tw-pan-x:  ;
+  --tw-pan-y:  ;
+  --tw-pinch-zoom:  ;
+  --tw-scroll-snap-strictness: proximity;
+  --tw-ordinal:  ;
+  --tw-slashed-zero:  ;
+  --tw-numeric-figure:  ;
+  --tw-numeric-spacing:  ;
+  --tw-numeric-fraction:  ;
+  --tw-ring-inset:  ;
+  --tw-ring-offset-width: 0px;
+  --tw-ring-offset-color: #fff;
+  --tw-ring-color: rgb(59 130 246 / 0.5);
+  --tw-ring-offset-shadow: 0 0 #0000;
+  --tw-ring-shadow: 0 0 #0000;
+  --tw-shadow: 0 0 #0000;
+  --tw-shadow-colored: 0 0 #0000;
+  --tw-blur:  ;
+  --tw-brightness:  ;
+  --tw-contrast:  ;
+  --tw-grayscale:  ;
+  --tw-hue-rotate:  ;
+  --tw-invert:  ;
+  --tw-saturate:  ;
+  --tw-sepia:  ;
+  --tw-drop-shadow:  ;
+  --tw-backdrop-blur:  ;
+  --tw-backdrop-brightness:  ;
+  --tw-backdrop-contrast:  ;
+  --tw-backdrop-grayscale:  ;
+  --tw-backdrop-hue-rotate:  ;
+  --tw-backdrop-invert:  ;
+  --tw-backdrop-opacity:  ;
+  --tw-backdrop-saturate:  ;
+  --tw-backdrop-sepia:  ;
+}
+
+::backdrop {
+  --tw-border-spacing-x: 0;
+  --tw-border-spacing-y: 0;
+  --tw-translate-x: 0;
+  --tw-translate-y: 0;
+  --tw-rotate: 0;
+  --tw-skew-x: 0;
+  --tw-skew-y: 0;
+  --tw-scale-x: 1;
+  --tw-scale-y: 1;
+  --tw-pan-x:  ;
+  --tw-pan-y:  ;
+  --tw-pinch-zoom:  ;
+  --tw-scroll-snap-strictness: proximity;
+  --tw-ordinal:  ;
+  --tw-slashed-zero:  ;
+  --tw-numeric-figure:  ;
+  --tw-numeric-spacing:  ;
+  --tw-numeric-fraction:  ;
+  --tw-ring-inset:  ;
+  --tw-ring-offset-width: 0px;
+  --tw-ring-offset-color: #fff;
+  --tw-ring-color: rgb(59 130 246 / 0.5);
+  --tw-ring-offset-shadow: 0 0 #0000;
+  --tw-ring-shadow: 0 0 #0000;
+  --tw-shadow: 0 0 #0000;
+  --tw-shadow-colored: 0 0 #0000;
+  --tw-blur:  ;
+  --tw-brightness:  ;
+  --tw-contrast:  ;
+  --tw-grayscale:  ;
+  --tw-hue-rotate:  ;
+  --tw-invert:  ;
+  --tw-saturate:  ;
+  --tw-sepia:  ;
+  --tw-drop-shadow:  ;
+  --tw-backdrop-blur:  ;
+  --tw-backdrop-brightness:  ;
+  --tw-backdrop-contrast:  ;
+  --tw-backdrop-grayscale:  ;
+  --tw-backdrop-hue-rotate:  ;
+  --tw-backdrop-invert:  ;
+  --tw-backdrop-opacity:  ;
+  --tw-backdrop-saturate:  ;
+  --tw-backdrop-sepia:  ;
+}
+
+.mb-2 {
+  margin-bottom: 0.5rem;
+}
+
+.ml-2 {
+  margin-left: 0.5rem;
+}
+
+.mr-2 {
+  margin-right: 0.5rem;
+}
+
+.mt-4 {
+  margin-top: 1rem;
+}
+
+.flex {
+  display: flex;
+}
+
+.w-\[26rem\] {
+  width: 26rem;
+}
+
+.w-full {
+  width: 100%;
+}
+
+.flex-col {
+  flex-direction: column;
+}
+
+.items-center {
+  align-items: center;
+}
+
+.justify-between {
+  justify-content: space-between;
+}
+
+.self-start {
+  align-self: flex-start;
+}
+
+.self-end {
+  align-self: flex-end;
+}
+
+.rounded-lg {
+  border-radius: 0.5rem;
+}
+
+.bg-blue-500 {
+  --tw-bg-opacity: 1;
+  background-color: rgb(59 130 246 / var(--tw-bg-opacity));
+}
+
+.bg-gray-200 {
+  --tw-bg-opacity: 1;
+  background-color: rgb(229 231 235 / var(--tw-bg-opacity));
+}
+
+.bg-gray-900 {
+  --tw-bg-opacity: 1;
+  background-color: rgb(17 24 39 / var(--tw-bg-opacity));
+}
+
+.bg-indigo-500 {
+  --tw-bg-opacity: 1;
+  background-color: rgb(99 102 241 / var(--tw-bg-opacity));
+}
+
+.bg-white {
+  --tw-bg-opacity: 1;
+  background-color: rgb(255 255 255 / var(--tw-bg-opacity));
+}
+
+.p-2 {
+  padding: 0.5rem;
+}
+
+.p-4 {
+  padding: 1rem;
+}
+
+.text-lg {
+  font-size: 1.125rem;
+  line-height: 1.75rem;
+}
+
+.text-sm {
+  font-size: 0.875rem;
+  line-height: 1.25rem;
+}
+
+.font-medium {
+  font-weight: 500;
+}
+
+.text-blue-500 {
+  --tw-text-opacity: 1;
+  color: rgb(59 130 246 / var(--tw-text-opacity));
+}
+
+.text-gray-700 {
+  --tw-text-opacity: 1;
+  color: rgb(55 65 81 / var(--tw-text-opacity));
+}
+
+.text-white {
+  --tw-text-opacity: 1;
+  color: rgb(255 255 255 / var(--tw-text-opacity));
+}
+
+.shadow {
+  --tw-shadow: 0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1);
+  --tw-shadow-colored: 0 1px 3px 0 var(--tw-shadow-color), 0 1px 2px -1px var(--tw-shadow-color);
+  box-shadow: var(--tw-ring-offset-shadow, 0 0 #0000), var(--tw-ring-shadow, 0 0 #0000), var(--tw-shadow);
+}
+
+#chat-container {
+  width: 500px;
+  height: 450px;
+  background-color: white;
+  padding: 10px;
+  overflow: auto;
+}
+
+.bg-gray-200 {
+  background-color: #edf2f7;
+}
+
+.bg-gray-900 {
+  background-color: #1a202c;
+}
+
+.rounded-lg {
+  border-radius: 0.5rem;
+}
+
+.shadow {
+  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
+}
+
+.text-gray-700 {
+  color: #4a5568;
+}
+
+.text-sm {
+  font-size: 0.875rem;
+}
+
+.p-4 {
+  padding: 1.5rem;
+}
+
+.hover\:text-blue-800:hover {
+  --tw-text-opacity: 1;
+  color: rgb(30 64 175 / var(--tw-text-opacity));
+}
+  
+
+

extensions/chrome/manifest.json

@@ -1,7 +1,7 @@
 {
   "name": "DocsGPT - Documentation AI butler",
   "version": "0.0.1",
-  "manifest_version": 2,
+  "manifest_version": 3,
   "description": "AI assistant for developers, that helps you answer your questions about the documentation you are reading.",
   "icons": {
     "16": "icons/icon16.png",
@@ -10,12 +10,19 @@
   },
   "default_locale": "en",
   "background": {
-    "page": "src/bg/background.html",
-    "persistent": true
+    "service_worker": "src/bg/service-worker.js"
+  },
+  "action": {
+    "default_title": "DocsGPT - Documentation AI butler",
+    "default_popup": "popup.html"
   },
   "permissions": ["activeTab", "storage"],
-  "browser_action": {
-    "default_popup": "popup.html"
-}
+  "host_permissions": [
+    "*://*/*"
+  ],
+  "content_scripts": [{
+    "js": ["popup.js"],
+    "matches": ["https://github.com/*"]
+  }]
 
 }

extensions/chrome/package.json

@@ -1,4 +1,19 @@
 {
+  "name": "docsgpt-chrome-extension",
+  "version": "0.0.1", 
+  "description": "DocsGPT - Documentation AI butler",
+  "main": "popup.js",
+  "author": "",
+  "license": "MIT",
+  "scripts": {
+    "dev": "npx tailwindcss -i ./styles.css -o ./dist/output.css --watch"
+  },
+  "keywords": [
+    "DocsGPT",
+    "Documentation",
+    "Chrome",
+    "extension"
+  ],
   "devDependencies": {
     "tailwindcss": "^3.2.4"
   }

extensions/chrome/popup.js

@@ -21,12 +21,12 @@ document.getElementById("message-form").addEventListener("submit", function(even
     }
 
     // send post request to server http://127.0.0.1:5000/ with message in json body
-    fetch('http://127.0.0.1:5000/api/answer', {
+    fetch('http://127.0.0.1:5001/api/answer', {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
       },
-      body: JSON.stringify({question: message}),
+      body: JSON.stringify({question: message, history: null}),
     })
     .then(response => response.json())
     .then(data => {

extensions/chrome/src/bg/background.html

@@ -1,6 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-  <script type="text/javascript" src="background.js"></script>
-</head>
-</html>

extensions/chrome/src/bg/background.js

@@ -1,22 +0,0 @@
-// if you checked "fancy-settings" in extensionizr.com, uncomment this lines
-
-// var settings = new Store("settings", {
-//     "sample_setting": "This is how you use Store.js to remember values"
-// });
-
-
-//example of using a message handler from the inject scripts
-chrome.extension.onMessage.addListener(
-  function(request, sender, sendResponse) {
-  	chrome.pageAction.show(sender.tab.id);
-    sendResponse();
-  });
-
-
-  chrome.runtime.onMessage.addListener(
-    function(request, sender, sendResponse) {
-      if (request.msg === "sendMessage") {
-        sendResponse({response: "Message received"});
-      }
-    });
-  

extensions/chrome/src/bg/service-worker.js

@@ -0,0 +1,12 @@
+// This is the service worker script, which executes in its own context
+// when the extension is installed or refreshed (or when you access its console).
+// It would correspond to the background script in chrome extensions v2.
+
+console.log("This prints to the console of the service worker (background script)");
+chrome.runtime.onMessage.addListener(
+    function(request, sender, sendResponse) {
+        if (request.msg === "sendMessage") {
+        sendResponse({response: "Message received"});
+        }
+    }
+);

extensions/discord/bot.py

@@ -0,0 +1,70 @@
+import os
+import re
+
+import discord
+import requests
+from discord.ext import commands
+import dotenv
+
+dotenv.load_dotenv()
+
+# Replace 'YOUR_BOT_TOKEN' with your bot's token
+TOKEN = os.getenv("DISCORD_TOKEN")
+PREFIX = '@docsgpt '
+BASE_API_URL = 'http://localhost:5001'
+
+intents = discord.Intents.default()
+intents.message_content = True
+
+bot = commands.Bot(command_prefix=PREFIX, intents=intents)
+
+
+def split_string(input_str):
+    pattern = r'<(.*?)>'
+    match = re.search(pattern, input_str)
+
+    if match:
+        content = match.group(1)
+        rest = input_str[:match.start()] + input_str[match.end():]
+        return content, rest.strip()
+    return None, input_str
+
+
+@bot.event
+async def on_ready():
+    print(f'{bot.user.name} has connected to Discord!')
+
+
+async def fetch_answer(question):
+    data = {
+        'sender': 'discord',
+        'question': question,
+        'history': ''
+    }
+    headers = {"Content-Type": "application/json",
+               "Accept": "application/json"}
+    response = requests.post(BASE_API_URL + '/api/answer', json=data, headers=headers)
+    if response.status_code == 200:
+        return response.json()['answer']
+    return 'Sorry, I could not fetch the answer.'
+
+
+@bot.event
+async def on_message(message):
+    if message.author == bot.user:
+        return
+
+    content = message.content.strip()
+    prefix, content = split_string(content)
+    if prefix is None:
+        return
+
+    part_prefix = "@"
+    if part_prefix in prefix:
+        answer = await fetch_answer(content)
+        await message.channel.send(answer)
+
+    await bot.process_commands(message)
+
+
+bot.run(TOKEN)

frontend/.env.development

@@ -0,0 +1,2 @@
+# Please put appropriate value
+VITE_API_HOST = http://localhost:5001

frontend/.env.production

@@ -0,0 +1 @@
+VITE_API_HOST = https://docsapi.arc53.com

frontend/index.html

@@ -4,6 +4,7 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>DocsGPT 🦖</title>
+      <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" />
   </head>
   <body>
     <div id="root" class="h-screen"></div>

frontend/package-lock.json

@@ -560,6 +560,122 @@
       "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.3.2.tgz",
       "integrity": "sha512-t54ONhl/h75X94SWsHGQ4G/ZrCEguKSRQr7DrjTciJXW0YU1QhlwYeycvK5JgkzlxmvrK7wq1NB/PLtHxoiDcA=="
     },
+    "@rollup/pluginutils": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.0.2.tgz",
+      "integrity": "sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==",
+      "dev": true,
+      "requires": {
+        "@types/estree": "^1.0.0",
+        "estree-walker": "^2.0.2",
+        "picomatch": "^2.3.1"
+      }
+    },
+    "@svgr/babel-plugin-add-jsx-attribute": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-add-jsx-attribute/-/babel-plugin-add-jsx-attribute-6.5.1.tgz",
+      "integrity": "sha512-9PYGcXrAxitycIjRmZB+Q0JaN07GZIWaTBIGQzfaZv+qr1n8X1XUEJ5rZ/vx6OVD9RRYlrNnXWExQXcmZeD/BQ==",
+      "dev": true
+    },
+    "@svgr/babel-plugin-remove-jsx-attribute": {
+      "version": "6.5.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-attribute/-/babel-plugin-remove-jsx-attribute-6.5.0.tgz",
+      "integrity": "sha512-8zYdkym7qNyfXpWvu4yq46k41pyNM9SOstoWhKlm+IfdCE1DdnRKeMUPsWIEO/DEkaWxJ8T9esNdG3QwQ93jBA==",
+      "dev": true
+    },
+    "@svgr/babel-plugin-remove-jsx-empty-expression": {
+      "version": "6.5.0",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-empty-expression/-/babel-plugin-remove-jsx-empty-expression-6.5.0.tgz",
+      "integrity": "sha512-NFdxMq3xA42Kb1UbzCVxplUc0iqSyM9X8kopImvFnB+uSDdzIHOdbs1op8ofAvVRtbg4oZiyRl3fTYeKcOe9Iw==",
+      "dev": true
+    },
+    "@svgr/babel-plugin-replace-jsx-attribute-value": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-replace-jsx-attribute-value/-/babel-plugin-replace-jsx-attribute-value-6.5.1.tgz",
+      "integrity": "sha512-8DPaVVE3fd5JKuIC29dqyMB54sA6mfgki2H2+swh+zNJoynC8pMPzOkidqHOSc6Wj032fhl8Z0TVn1GiPpAiJg==",
+      "dev": true
+    },
+    "@svgr/babel-plugin-svg-dynamic-title": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-dynamic-title/-/babel-plugin-svg-dynamic-title-6.5.1.tgz",
+      "integrity": "sha512-FwOEi0Il72iAzlkaHrlemVurgSQRDFbk0OC8dSvD5fSBPHltNh7JtLsxmZUhjYBZo2PpcU/RJvvi6Q0l7O7ogw==",
+      "dev": true
+    },
+    "@svgr/babel-plugin-svg-em-dimensions": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-em-dimensions/-/babel-plugin-svg-em-dimensions-6.5.1.tgz",
+      "integrity": "sha512-gWGsiwjb4tw+ITOJ86ndY/DZZ6cuXMNE/SjcDRg+HLuCmwpcjOktwRF9WgAiycTqJD/QXqL2f8IzE2Rzh7aVXA==",
+      "dev": true
+    },
+    "@svgr/babel-plugin-transform-react-native-svg": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-react-native-svg/-/babel-plugin-transform-react-native-svg-6.5.1.tgz",
+      "integrity": "sha512-2jT3nTayyYP7kI6aGutkyfJ7UMGtuguD72OjeGLwVNyfPRBD8zQthlvL+fAbAKk5n9ZNcvFkp/b1lZ7VsYqVJg==",
+      "dev": true
+    },
+    "@svgr/babel-plugin-transform-svg-component": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-svg-component/-/babel-plugin-transform-svg-component-6.5.1.tgz",
+      "integrity": "sha512-a1p6LF5Jt33O3rZoVRBqdxL350oge54iZWHNI6LJB5tQ7EelvD/Mb1mfBiZNAan0dt4i3VArkFRjA4iObuNykQ==",
+      "dev": true
+    },
+    "@svgr/babel-preset": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/babel-preset/-/babel-preset-6.5.1.tgz",
+      "integrity": "sha512-6127fvO/FF2oi5EzSQOAjo1LE3OtNVh11R+/8FXa+mHx1ptAaS4cknIjnUA7e6j6fwGGJ17NzaTJFUwOV2zwCw==",
+      "dev": true,
+      "requires": {
+        "@svgr/babel-plugin-add-jsx-attribute": "^6.5.1",
+        "@svgr/babel-plugin-remove-jsx-attribute": "*",
+        "@svgr/babel-plugin-remove-jsx-empty-expression": "*",
+        "@svgr/babel-plugin-replace-jsx-attribute-value": "^6.5.1",
+        "@svgr/babel-plugin-svg-dynamic-title": "^6.5.1",
+        "@svgr/babel-plugin-svg-em-dimensions": "^6.5.1",
+        "@svgr/babel-plugin-transform-react-native-svg": "^6.5.1",
+        "@svgr/babel-plugin-transform-svg-component": "^6.5.1"
+      }
+    },
+    "@svgr/core": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/core/-/core-6.5.1.tgz",
+      "integrity": "sha512-/xdLSWxK5QkqG524ONSjvg3V/FkNyCv538OIBdQqPNaAta3AsXj/Bd2FbvR87yMbXO2hFSWiAe/Q6IkVPDw+mw==",
+      "dev": true,
+      "requires": {
+        "@babel/core": "^7.19.6",
+        "@svgr/babel-preset": "^6.5.1",
+        "@svgr/plugin-jsx": "^6.5.1",
+        "camelcase": "^6.2.0",
+        "cosmiconfig": "^7.0.1"
+      }
+    },
+    "@svgr/hast-util-to-babel-ast": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/hast-util-to-babel-ast/-/hast-util-to-babel-ast-6.5.1.tgz",
+      "integrity": "sha512-1hnUxxjd83EAxbL4a0JDJoD3Dao3hmjvyvyEV8PzWmLK3B9m9NPlW7GKjFyoWE8nM7HnXzPcmmSyOW8yOddSXw==",
+      "dev": true,
+      "requires": {
+        "@babel/types": "^7.20.0",
+        "entities": "^4.4.0"
+      }
+    },
+    "@svgr/plugin-jsx": {
+      "version": "6.5.1",
+      "resolved": "https://registry.npmjs.org/@svgr/plugin-jsx/-/plugin-jsx-6.5.1.tgz",
+      "integrity": "sha512-+UdQxI3jgtSjCykNSlEMuy1jSRQlGC7pqBCPvkG/2dATdWo082zHTTK3uhnAju2/6XpE6B5mZ3z4Z8Ns01S8Gw==",
+      "dev": true,
+      "requires": {
+        "@babel/core": "^7.19.6",
+        "@svgr/babel-preset": "^6.5.1",
+        "@svgr/hast-util-to-babel-ast": "^6.5.1",
+        "svg-parser": "^2.0.4"
+      }
+    },
+    "@types/estree": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.0.tgz",
+      "integrity": "sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ==",
+      "dev": true
+    },
     "@types/hoist-non-react-statics": {
       "version": "3.3.1",
       "resolved": "https://registry.npmjs.org/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.1.tgz",
@@ -581,6 +697,12 @@
       "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==",
       "dev": true
     },
+    "@types/parse-json": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
+      "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==",
+      "dev": true
+    },
     "@types/prop-types": {
       "version": "15.7.5",
       "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz",
@@ -983,6 +1105,11 @@
       "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
       "dev": true
     },
+    "attr-accept": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz",
+      "integrity": "sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg=="
+    },
     "autoprefixer": {
       "version": "10.4.13",
       "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz",
@@ -1097,6 +1224,12 @@
       "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
       "dev": true
     },
+    "camelcase": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
+      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
+      "dev": true
+    },
     "camelcase-css": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz",
@@ -1219,6 +1352,19 @@
       "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==",
       "dev": true
     },
+    "cosmiconfig": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz",
+      "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==",
+      "dev": true,
+      "requires": {
+        "@types/parse-json": "^4.0.0",
+        "import-fresh": "^3.2.1",
+        "parse-json": "^5.0.0",
+        "path-type": "^4.0.0",
+        "yaml": "^1.10.0"
+      }
+    },
     "cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -1331,6 +1477,21 @@
       "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==",
       "dev": true
     },
+    "entities": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz",
+      "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==",
+      "dev": true
+    },
+    "error-ex": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
+      "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
+      "dev": true,
+      "requires": {
+        "is-arrayish": "^0.2.1"
+      }
+    },
     "es-abstract": {
       "version": "1.21.1",
       "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.21.1.tgz",
@@ -1917,6 +2078,12 @@
       "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
       "dev": true
     },
+    "estree-walker": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz",
+      "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==",
+      "dev": true
+    },
     "esutils": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
@@ -2006,6 +2173,21 @@
         "flat-cache": "^3.0.4"
       }
     },
+    "file-selector": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.6.0.tgz",
+      "integrity": "sha512-QlZ5yJC0VxHxQQsQhXvBaC7VRJ2uaxTf+Tfpu4Z/OcVQJVpZO+DGU0rkoVW5ce2SccxugvpBJoMvUs59iILYdw==",
+      "requires": {
+        "tslib": "^2.4.0"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.5.0",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
+          "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
+        }
+      }
+    },
     "fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -2335,6 +2517,12 @@
         "is-typed-array": "^1.1.10"
       }
     },
+    "is-arrayish": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
+      "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
+      "dev": true
+    },
     "is-bigint": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz",
@@ -2532,6 +2720,12 @@
       "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==",
       "dev": true
     },
+    "json-parse-even-better-errors": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
+      "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
+      "dev": true
+    },
     "json-schema-traverse": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
@@ -2576,6 +2770,12 @@
       "integrity": "sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg==",
       "dev": true
     },
+    "lines-and-columns": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
+      "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==",
+      "dev": true
+    },
     "lint-staged": {
       "version": "13.1.1",
       "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-13.1.1.tgz",
@@ -2908,8 +3108,7 @@
     "object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
-      "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
-      "dev": true
+      "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="
     },
     "object-hash": {
       "version": "3.0.0",
@@ -3052,6 +3251,18 @@
         "callsites": "^3.0.0"
       }
     },
+    "parse-json": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz",
+      "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==",
+      "dev": true,
+      "requires": {
+        "@babel/code-frame": "^7.0.0",
+        "error-ex": "^1.3.1",
+        "json-parse-even-better-errors": "^2.3.0",
+        "lines-and-columns": "^1.1.6"
+      }
+    },
     "path-exists": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
@@ -3203,7 +3414,6 @@
       "version": "15.8.1",
       "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
       "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
-      "dev": true,
       "requires": {
         "loose-envify": "^1.4.0",
         "object-assign": "^4.1.1",
@@ -3245,6 +3455,16 @@
         "scheduler": "^0.23.0"
       }
     },
+    "react-dropzone": {
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.2.3.tgz",
+      "integrity": "sha512-O3om8I+PkFKbxCukfIR3QAGftYXDZfOE2N1mr/7qebQJHs7U+/RSL/9xomJNpRg9kM5h9soQSdf0Gc7OHF5Fug==",
+      "requires": {
+        "attr-accept": "^2.2.2",
+        "file-selector": "^0.6.0",
+        "prop-types": "^15.8.1"
+      }
+    },
     "react-is": {
       "version": "16.13.1",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
@@ -3652,6 +3872,12 @@
       "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
       "dev": true
     },
+    "svg-parser": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz",
+      "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==",
+      "dev": true
+    },
     "tailwindcss": {
       "version": "3.2.4",
       "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.2.4.tgz",
@@ -3835,6 +4061,16 @@
         "rollup": "^3.10.0"
       }
     },
+    "vite-plugin-svgr": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/vite-plugin-svgr/-/vite-plugin-svgr-2.4.0.tgz",
+      "integrity": "sha512-q+mJJol6ThvqkkJvvVFEndI4EaKIjSI0I3jNFgSoC9fXAz1M7kYTVUin8fhUsFojFDKZ9VHKtX6NXNaOLpbsHA==",
+      "dev": true,
+      "requires": {
+        "@rollup/pluginutils": "^5.0.2",
+        "@svgr/core": "^6.5.1"
+      }
+    },
     "which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

frontend/package.json

@@ -23,6 +23,7 @@
     "@vercel/analytics": "^0.1.10",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "react-dropzone": "^14.2.3",
     "react-redux": "^8.0.5",
     "react-router-dom": "^6.8.1"
   },
@@ -49,6 +50,7 @@
     "prettier-plugin-tailwindcss": "^0.2.2",
     "tailwindcss": "^3.2.4",
     "typescript": "^4.9.5",
-    "vite": "^4.1.0"
+    "vite": "^4.1.0",
+    "vite-plugin-svgr": "^2.4.0"
   }
 }

frontend/src/Navigation.tsx

@@ -1,12 +1,14 @@
-import { useRef, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import { NavLink } from 'react-router-dom';
 import Arrow1 from './assets/arrow.svg';
 import Arrow2 from './assets/dropdown-arrow.svg';
+import Exit from './assets/exit.svg';
 import Message from './assets/message.svg';
 import Hamburger from './assets/hamburger.svg';
 import Key from './assets/key.svg';
 import Info from './assets/info.svg';
 import Link from './assets/link.svg';
+import UploadIcon from './assets/upload.svg';
 import { ActiveState } from './models/misc';
 import APIKeyModal from './preferences/APIKeyModal';
 import SelectDocsModal from './preferences/SelectDocsModal';
@@ -19,6 +21,8 @@ import {
   setSelectedDocs,
 } from './preferences/preferenceSlice';
 import { useOutsideAlerter } from './hooks';
+import Upload from './upload/Upload';
+import { Doc } from './preferences/preferenceApi';
 
 export default function Navigation({
   navState,
@@ -42,7 +46,28 @@ export default function Navigation({
   const [selectedDocsModalState, setSelectedDocsModalState] =
     useState<ActiveState>(isSelectedDocsSet ? 'INACTIVE' : 'ACTIVE');
 
+  const [uploadModalState, setUploadModalState] =
+    useState<ActiveState>('INACTIVE');
+
   const navRef = useRef(null);
+  const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+
+  const handleDeleteClick = (index: number, doc: Doc) => {
+    const docPath = 'indexes/' + 'local' + '/' + doc.name;
+
+    fetch(`${apiHost}/api/delete_old?path=${docPath}`, {
+      method: 'GET',
+    })
+      .then(() => {
+        // remove the image element from the DOM
+        const imageElement = document.querySelector(
+          `#img-${index}`,
+        ) as HTMLElement;
+        const parentElement = imageElement.parentNode as HTMLElement;
+        parentElement.parentNode?.removeChild(parentElement);
+      })
+      .catch((error) => console.error(error));
+  };
   useOutsideAlerter(
     navRef,
     () => {
@@ -58,6 +83,19 @@ export default function Navigation({
     [navState, isDocsListOpen, apiKeyModalState],
   );
 
+  /*
+    Needed to fix bug where if mobile nav was closed and then window was resized to desktop, nav would still be closed but the button to open would be gone, as per #1 on issue #146
+  */
+  useEffect(() => {
+    window.addEventListener('resize', () => {
+      if (window.matchMedia('(min-width: 768px)').matches) {
+        setNavState('ACTIVE');
+      } else {
+        setNavState('INACTIVE');
+      }
+    });
+  }, []);
+
   return (
     <>
       <div
@@ -66,7 +104,7 @@ export default function Navigation({
           navState === 'INACTIVE' && '-ml-96 md:-ml-[14rem]'
         } duration-20 fixed z-20 flex h-full w-72 flex-col border-r-2 bg-gray-50 transition-all`}
       >
-        <div className={'h-16 w-full border-b-2'}>
+        <div className={'visible h-16 w-full border-b-2 md:hidden'}>
           <button
             className="float-right mr-5 mt-5 h-5 w-5"
             onClick={() =>
@@ -95,8 +133,8 @@ export default function Navigation({
         </NavLink>
 
         <div className="flex-grow border-b-2 border-gray-100"></div>
-        <div className="flex flex-grow flex-col-reverse border-b-2">
-          <div className="relative my-4 px-6">
+        <div className="flex flex-col-reverse border-b-2">
+          <div className="relative my-4 flex gap-2 px-2">
             <div
               className="flex h-12 w-full cursor-pointer justify-between rounded-md border-2 bg-white"
               onClick={() => setIsDocsListOpen(!isDocsListOpen)}
@@ -114,11 +152,16 @@ export default function Navigation({
                 } mr-3 w-3 transition-all`}
               />
             </div>
+            <img
+              className="mt-2 h-9 w-9 hover:cursor-pointer"
+              src={UploadIcon}
+              onClick={() => setUploadModalState('ACTIVE')}
+            ></img>
             {isDocsListOpen && (
-              <div className="absolute top-12 left-0 right-0 mx-6 max-h-52 overflow-y-scroll bg-white shadow-lg">
+              <div className="absolute top-12 left-0 right-6 ml-2 mr-4 max-h-52 overflow-y-scroll bg-white shadow-lg">
                 {docs ? (
                   docs.map((doc, index) => {
-                    if (doc.model) {
+                    if (doc.model === 'openai_text-embedding-ada-002') {
                       return (
                         <div
                           key={index}
@@ -126,11 +169,23 @@ export default function Navigation({
                             dispatch(setSelectedDocs(doc));
                             setIsDocsListOpen(false);
                           }}
-                          className="h-10 w-full cursor-pointer border-x-2 border-b-2 hover:bg-gray-100"
+                          className="flex h-10 w-full cursor-pointer items-center justify-between border-x-2 border-b-2 hover:bg-gray-100"
                         >
-                          <p className="ml-5 py-3">
+                          <p className="ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3">
                             {doc.name} {doc.version}
                           </p>
+                          {doc.location === 'local' ? (
+                            <img
+                              src={Exit}
+                              alt="Exit"
+                              className="mr-4 h-3 w-3 cursor-pointer hover:opacity-50"
+                              id={`img-${index}`}
+                              onClick={(event) => {
+                                event.stopPropagation();
+                                handleDeleteClick(index, doc);
+                              }}
+                            />
+                          ) : null}
                         </div>
                       );
                     }
@@ -143,7 +198,7 @@ export default function Navigation({
               </div>
             )}
           </div>
-          <p className="ml-6 font-bold text-jet">Source Docs</p>
+          <p className="ml-6 mt-3 font-bold text-jet">Source Docs</p>
         </div>
         <div className="flex flex-col gap-2 border-b-2 py-2">
           <div
@@ -209,6 +264,10 @@ export default function Navigation({
         setModalState={setApiKeyModalState}
         isCancellable={isApiKeySet}
       />
+      <Upload
+        modalState={uploadModalState}
+        setModalState={setUploadModalState}
+      ></Upload>
     </>
   );
 }

frontend/src/assets/dislike.svg

@@ -0,0 +1,4 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.37776 10.1001V12.9C6.37776 13.457 6.599 13.9911 6.99282 14.3849C7.38664 14.7788 7.92077 15 8.47772 15L11.2777 8.70011V1.00025H3.38181C3.04419 0.996436 2.71656 1.11477 2.45929 1.33344C2.20203 1.55212 2.03246 1.8564 1.98184 2.19023L1.01585 8.49012C0.985398 8.69076 0.998931 8.89563 1.05551 9.09053C1.1121 9.28543 1.21038 9.46569 1.34355 9.61884C1.47671 9.77198 1.64159 9.89434 1.82674 9.97744C2.01189 10.0605 2.2129 10.1024 2.41583 10.1001H6.37776ZM11.2777 1.00025H13.1466C13.5428 0.993247 13.9277 1.13195 14.2284 1.39002C14.5291 1.64809 14.7245 2.00758 14.7776 2.40023V7.30014C14.7245 7.69279 14.5291 8.05227 14.2284 8.31035C13.9277 8.56842 13.5428 8.70712 13.1466 8.70011H11.2777" fill="white"/>
+<path d="M11.2777 8.70011L8.47772 15C7.92077 15 7.38664 14.7788 6.99282 14.3849C6.599 13.9911 6.37776 13.457 6.37776 12.9V10.1001H2.41583C2.2129 10.1024 2.01189 10.0605 1.82674 9.97744C1.64159 9.89434 1.47671 9.77198 1.34355 9.61884C1.21038 9.46569 1.1121 9.28543 1.05551 9.09053C0.998931 8.89563 0.985398 8.69076 1.01585 8.49012L1.98184 2.19023C2.03246 1.8564 2.20203 1.55212 2.45929 1.33344C2.71656 1.11477 3.04419 0.996436 3.38181 1.00025H11.2777M11.2777 8.70011V1.00025M11.2777 8.70011H13.1466C13.5428 8.70712 13.9277 8.56842 14.2284 8.31035C14.5291 8.05227 14.7245 7.69279 14.7776 7.30014V2.40023C14.7245 2.00758 14.5291 1.64809 14.2284 1.39002C13.9277 1.13195 13.5428 0.993247 13.1466 1.00025H11.2777" stroke="current" stroke-width="1.4" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

frontend/src/assets/like.svg

@@ -0,0 +1,4 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="current" xmlns="http://www.w3.org/2000/svg">
+<path d="M9.39995 5.89997V3.09999C9.39995 2.54304 9.1787 2.0089 8.78487 1.61507C8.39105 1.22125 7.85691 1 7.29996 1L4.49998 7.29996V14.9999H12.3959C12.7336 15.0037 13.0612 14.8854 13.3185 14.6667C13.5757 14.448 13.7453 14.1437 13.7959 13.8099L14.7619 7.50996C14.7924 7.30931 14.7788 7.10444 14.7222 6.90954C14.6657 6.71464 14.5674 6.53437 14.4342 6.38123C14.301 6.22808 14.1362 6.10572 13.951 6.02262C13.7659 5.93952 13.5649 5.89767 13.3619 5.89997H9.39995ZM4.49998 14.9999H2.39999C2.02869 14.9999 1.6726 14.8524 1.41005 14.5899C1.1475 14.3273 1 13.9712 1 13.5999V8.69995C1 8.32865 1.1475 7.97256 1.41005 7.71001C1.6726 7.44746 2.02869 7.29996 2.39999 7.29996H4.49998" fill="white"/>
+<path d="M4.49998 7.29996L7.29996 1C7.85691 1 8.39105 1.22125 8.78487 1.61507C9.1787 2.0089 9.39995 2.54304 9.39995 3.09999V5.89997H13.3619C13.5649 5.89767 13.7659 5.93952 13.951 6.02262C14.1362 6.10572 14.301 6.22808 14.4342 6.38123C14.5674 6.53437 14.6657 6.71464 14.7223 6.90954C14.7788 7.10444 14.7924 7.30931 14.7619 7.50996L13.7959 13.8099C13.7453 14.1437 13.5757 14.448 13.3185 14.6667C13.0612 14.8854 12.7336 15.0037 12.3959 14.9999H4.49998M4.49998 7.29996V14.9999M4.49998 7.29996H2.39999C2.02869 7.29996 1.6726 7.44746 1.41005 7.71001C1.1475 7.97256 1 8.32865 1 8.69995V13.5999C1 13.9712 1.1475 14.3273 1.41005 14.5899C1.6726 14.8524 2.02869 14.9999 2.39999 14.9999H4.49998" stroke="current" stroke-width="1.39999" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

frontend/src/assets/upload.svg

@@ -0,0 +1,3 @@
+<svg width="24" height="16" viewBox="0 0 24 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M19.35 6.04C18.67 2.59 15.64 0 12 0C9.11 0 6.6 1.64 5.35 4.04C2.34 4.36 0 6.91 0 10C0 13.31 2.69 16 6 16H19C21.76 16 24 13.76 24 11C24 8.36 21.95 6.22 19.35 6.04ZM14 9V13H10V9H7L12 4L17 9H14Z" fill="black" fill-opacity="0.54"/>
+</svg>

frontend/src/conversation/Conversation.tsx

@@ -1,19 +1,22 @@
-import { useEffect, useRef } from 'react';
+import { Fragment, useEffect, useRef } from 'react';
 import { useDispatch, useSelector } from 'react-redux';
 import Hero from '../Hero';
 import { AppDispatch } from '../store';
 import ConversationBubble from './ConversationBubble';
 import {
-  addMessage,
+  addQuery,
   fetchAnswer,
-  selectConversation,
+  selectQueries,
   selectStatus,
+  updateQuery,
 } from './conversationSlice';
 import Send from './../assets/send.svg';
 import Spinner from './../assets/spinner.svg';
+import { FEEDBACK, Query } from './conversationModels';
+import { sendFeedback } from './conversationApi';
 
 export default function Conversation() {
-  const messages = useSelector(selectConversation);
+  const queries = useSelector(selectQueries);
   const status = useSelector(selectStatus);
   const dispatch = useDispatch<AppDispatch>();
   const endMessageRef = useRef<HTMLDivElement>(null);
@@ -21,35 +24,78 @@ export default function Conversation() {
 
   useEffect(
     () => endMessageRef?.current?.scrollIntoView({ behavior: 'smooth' }),
-    [messages],
+    [queries.length, queries[queries.length - 1]],
   );
 
   const handleQuestion = (question: string) => {
-    dispatch(addMessage({ text: question, type: 'QUESTION' }));
+    dispatch(addQuery({ prompt: question }));
     dispatch(fetchAnswer({ question }));
   };
 
+  const handleFeedback = (query: Query, feedback: FEEDBACK, index: number) => {
+    const prevFeedback = query.feedback;
+    dispatch(updateQuery({ index, query: { feedback } }));
+    sendFeedback(query.prompt, query.response!, feedback).catch(() =>
+      dispatch(updateQuery({ index, query: { feedback: prevFeedback } })),
+    );
+  };
+
+  const prepResponseView = (query: Query, index: number) => {
+    let responseView;
+    if (query.error) {
+      responseView = (
+        <ConversationBubble
+          ref={endMessageRef}
+          className={`${index === queries.length - 1 ? 'mb-24' : 'mb-7'}`}
+          key={`${index}ERROR`}
+          message={query.error}
+          type="ERROR"
+        ></ConversationBubble>
+      );
+    } else if (query.response) {
+      responseView = (
+        <ConversationBubble
+          ref={endMessageRef}
+          className={`${index === queries.length - 1 ? 'mb-24' : 'mb-7'}`}
+          key={`${index}ANSWER`}
+          message={query.response}
+          type={'ANSWER'}
+          feedback={query.feedback}
+          handleFeedback={(feedback: FEEDBACK) =>
+            handleFeedback(query, feedback, index)
+          }
+        ></ConversationBubble>
+      );
+    }
+    return responseView;
+  };
+
   return (
     <div className="flex justify-center p-6">
-      {messages.length > 0 && (
-        <div className="mt-20 flex w-10/12 flex-col transition-all md:w-1/2">
-          {messages.map((message, index) => {
+      {queries.length > 0 && (
+        <div className="mt-20 flex w-10/12 flex-col transition-all md:w-3/4">
+          {queries.map((query, index) => {
             return (
-              <ConversationBubble
-                ref={index === messages.length - 1 ? endMessageRef : null}
-                className={`${
-                  index === messages.length - 1 ? 'mb-24' : 'mb-7'
-                }`}
-                key={index}
-                message={message.text}
-                type={message.type}
-              ></ConversationBubble>
+              <Fragment key={index}>
+                <ConversationBubble
+                  ref={endMessageRef}
+                  className={`${
+                    index === queries.length - 1 && status === 'loading'
+                      ? 'mb-24'
+                      : 'mb-7'
+                  }`}
+                  key={`${index}QUESTION`}
+                  message={query.prompt}
+                  type="QUESTION"
+                ></ConversationBubble>
+                {prepResponseView(query, index)}
+              </Fragment>
             );
           })}
         </div>
       )}
-      {messages.length === 0 && <Hero className="mt-24 md:mt-52"></Hero>}
-      <div className="fixed bottom-6 flex w-10/12 flex-col items-end self-center md:w-[50%]">
+      {queries.length === 0 && <Hero className="mt-24 md:mt-52"></Hero>}
+      <div className="fixed bottom-0 flex w-10/12 flex-col items-end self-center md:w-[50%]">
         <div className="flex w-full">
           <div
             ref={inputRef}
@@ -85,7 +131,7 @@ export default function Conversation() {
             </div>
           )}
         </div>
-        <p className="mt-3 w-10/12 self-center text-center text-xs text-gray-2000">
+        <p className="w-[100vw] self-center bg-white p-5 text-center text-xs text-gray-2000">
           This is a chatbot that uses the GPT-3, Faiss and LangChain to answer
           questions.
         </p>

frontend/src/conversation/ConversationBubble.tsx

@@ -1,51 +1,92 @@
-import { forwardRef } from 'react';
+import { forwardRef, useState } from 'react';
 import Avatar from '../Avatar';
-import { MESSAGE_TYPE } from './conversationModels';
+import { FEEDBACK, MESSAGE_TYPE } from './conversationModels';
 import Alert from './../assets/alert.svg';
+import { ReactComponent as Like } from './../assets/like.svg';
+import { ReactComponent as Dislike } from './../assets/dislike.svg';
 
 const ConversationBubble = forwardRef<
   HTMLDivElement,
   {
     message: string;
     type: MESSAGE_TYPE;
-    className: string;
+    className?: string;
+    feedback?: FEEDBACK;
+    handleFeedback?: (feedback: FEEDBACK) => void;
   }
->(function ConversationBubble({ message, type, className }, ref) {
-  return (
-    <div
-      ref={ref}
-      className={`flex ${
-        type === 'ANSWER'
-          ? 'self-start '
-          : type === 'ERROR'
-          ? 'self-start'
-          : 'flex-row-reverse self-end '
-      }  ${className}`}
-    >
-      <Avatar
-        className="mt-4 text-2xl"
-        avatar={type === 'QUESTION' ? '🧑‍💻' : '🦖'}
-      ></Avatar>
-      <div
-        className={`${
-          type === 'QUESTION'
-            ? ' mr-2 ml-10 bg-blue-1000 py-5 px-5 text-white'
-            : ' ml-2 mr-10 bg-gray-1000 py-5 px-5'
-        } flex items-center rounded-3xl ${
-          type === 'ERROR'
-            ? 'rounded-lg border border-red-2000 bg-red-1000 p-2 text-red-3000'
-            : ''
-        }`}
-      >
-        {type === 'ERROR' && (
-          <img src={Alert} alt="alert" className="mr-2 inline" />
-        )}
-        <p className="whitespace-pre-wrap break-words">{message}</p>
+>(function ConversationBubble(
+  { message, type, className, feedback, handleFeedback },
+  ref,
+) {
+  const [showFeedback, setShowFeedback] = useState(false);
+  let bubble;
+
+  if (type === 'QUESTION') {
+    bubble = (
+      <div ref={ref} className={`flex flex-row-reverse self-end ${className}`}>
+        <Avatar className="mt-4 text-2xl" avatar="🧑‍💻"></Avatar>
+        <div className="mr-2 ml-10 flex items-center rounded-3xl bg-blue-1000 py-5 px-5 text-white">
+          <p className="whitespace-pre-wrap break-words">{message}</p>
+        </div>
       </div>
-    </div>
-  );
+    );
+  } else {
+    bubble = (
+      <div
+        ref={ref}
+        className={`flex self-start ${className}`}
+        onMouseEnter={() => setShowFeedback(true)}
+        onMouseLeave={() => setShowFeedback(false)}
+      >
+        <Avatar className="mt-4 text-2xl" avatar="🦖"></Avatar>
+        <div
+          className={`ml-2 mr-5 flex items-center rounded-3xl bg-gray-1000 py-5 px-5 ${
+            type === 'ERROR'
+              ? ' rounded-lg border border-red-2000 bg-red-1000 p-2 text-red-3000'
+              : ''
+          }`}
+        >
+          {type === 'ERROR' && (
+            <img src={Alert} alt="alert" className="mr-2 inline" />
+          )}
+          <p className="whitespace-pre-wrap break-words">{message}</p>
+        </div>
+        <div
+          className={`mr-2 flex items-center justify-center ${
+            feedback === 'LIKE' || (type !== 'ERROR' && showFeedback)
+              ? ''
+              : 'md:invisible'
+          }`}
+        >
+          <Like
+            className={`cursor-pointer ${
+              feedback === 'LIKE'
+                ? 'fill-blue-1000 stroke-blue-1000'
+                : 'fill-none  stroke-gray-4000 hover:fill-gray-4000'
+            }`}
+            onClick={() => handleFeedback?.('LIKE')}
+          ></Like>
+        </div>
+        <div
+          className={`mr-10 flex items-center justify-center ${
+            feedback === 'DISLIKE' || (type !== 'ERROR' && showFeedback)
+              ? ''
+              : 'md:invisible'
+          }`}
+        >
+          <Dislike
+            className={`cursor-pointer ${
+              feedback === 'DISLIKE'
+                ? 'fill-red-2000 stroke-red-2000'
+                : 'fill-none  stroke-gray-4000 hover:fill-gray-4000'
+            }`}
+            onClick={() => handleFeedback?.('DISLIKE')}
+          ></Dislike>
+        </div>
+      </div>
+    );
+  }
+  return bubble;
 });
 
 export default ConversationBubble;
-
-// TODO : split question and answer into two diff JSX

frontend/src/conversation/conversationApi.ts

@@ -1,6 +1,8 @@
-import { Answer } from './conversationModels';
+import { Answer, FEEDBACK } from './conversationModels';
 import { Doc } from '../preferences/preferenceApi';
 
+const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+
 export function fetchAnswerApi(
   question: string,
   apiKey: string,
@@ -11,19 +13,20 @@ export function fetchAnswerApi(
     namePath = '.project';
   }
 
-  const docPath =
-    selectedDocs.name === 'default'
-      ? 'default'
-      : selectedDocs.language +
-        '/' +
-        namePath +
-        '/' +
-        selectedDocs.version +
-        '/' +
-        selectedDocs.model +
-        '/';
-
-  const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+  let docPath = 'default';
+  if (selectedDocs.location === 'local') {
+    docPath = 'local' + '/' + selectedDocs.name + '/';
+  } else if (selectedDocs.location === 'remote') {
+    docPath =
+      selectedDocs.language +
+      '/' +
+      namePath +
+      '/' +
+      selectedDocs.version +
+      '/' +
+      selectedDocs.model +
+      '/';
+  }
 
   return fetch(apiHost + '/api/answer', {
     method: 'POST',
@@ -51,8 +54,26 @@ export function fetchAnswerApi(
     });
 }
 
-function getRandomInt(min: number, max: number) {
-  min = Math.ceil(min);
-  max = Math.floor(max);
-  return Math.floor(Math.random() * (max - min) + min); // The maximum is exclusive and the minimum is inclusive
+export function sendFeedback(
+  prompt: string,
+  response: string,
+  feedback: FEEDBACK,
+) {
+  return fetch(`${apiHost}/api/feedback`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      question: prompt,
+      answer: response,
+      feedback: feedback,
+    }),
+  }).then((response) => {
+    if (response.ok) {
+      return Promise.resolve();
+    } else {
+      return Promise.reject();
+    }
+  });
 }

frontend/src/conversation/conversationModels.ts

@@ -1,5 +1,6 @@
 export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER' | 'ERROR';
 export type Status = 'idle' | 'loading' | 'failed';
+export type FEEDBACK = 'LIKE' | 'DISLIKE';
 
 export interface Message {
   text: string;
@@ -7,7 +8,7 @@ export interface Message {
 }
 
 export interface ConversationState {
-  conversation: Message[];
+  queries: Query[];
   status: Status;
 }
 
@@ -16,3 +17,10 @@ export interface Answer {
   query: string;
   result: string;
 }
+
+export interface Query {
+  prompt: string;
+  response?: string;
+  feedback?: FEEDBACK;
+  error?: string;
+}

frontend/src/conversation/conversationSlice.ts

@@ -1,10 +1,10 @@
 import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';
 import store from '../store';
 import { fetchAnswerApi } from './conversationApi';
-import { Answer, ConversationState, Message } from './conversationModels';
+import { Answer, ConversationState, Query } from './conversationModels';
 
 const initialState: ConversationState = {
-  conversation: [],
+  queries: [],
   status: 'idle',
 };
 
@@ -27,8 +27,18 @@ export const conversationSlice = createSlice({
   name: 'conversation',
   initialState,
   reducers: {
-    addMessage(state, action: PayloadAction<Message>) {
-      state.conversation.push(action.payload);
+    addQuery(state, action: PayloadAction<Query>) {
+      state.queries.push(action.payload);
+    },
+    updateQuery(
+      state,
+      action: PayloadAction<{ index: number; query: Partial<Query> }>,
+    ) {
+      const index = action.payload.index;
+      state.queries[index] = {
+        ...state.queries[index],
+        ...action.payload.query,
+      };
     },
   },
   extraReducers(builder) {
@@ -38,27 +48,22 @@ export const conversationSlice = createSlice({
       })
       .addCase(fetchAnswer.fulfilled, (state, action) => {
         state.status = 'idle';
-        state.conversation.push({
-          text: action.payload.answer,
-          type: 'ANSWER',
-        });
+        state.queries[state.queries.length - 1].response =
+          action.payload.answer;
       })
       .addCase(fetchAnswer.rejected, (state, action) => {
         state.status = 'failed';
-        state.conversation.push({
-          text: 'Something went wrong. Please try again later.',
-          type: 'ERROR',
-        });
+        state.queries[state.queries.length - 1].error =
+          'Something went wrong. Please try again later.';
       });
   },
 });
 
 type RootState = ReturnType<typeof store.getState>;
 
-export const selectConversation = (state: RootState) =>
-  state.conversation.conversation;
+export const selectQueries = (state: RootState) => state.conversation.queries;
 
 export const selectStatus = (state: RootState) => state.conversation.status;
 
-export const { addMessage } = conversationSlice.actions;
+export const { addQuery, updateQuery } = conversationSlice.actions;
 export default conversationSlice.reducer;

frontend/src/preferences/preferenceApi.ts

@@ -1,5 +1,6 @@
 // not all properties in Doc are going to be present. Make some optional
 export type Doc = {
+  location: string;
   name: string;
   language: string;
   version: string;
@@ -13,9 +14,10 @@ export type Doc = {
 //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
 export async function getDocs(): Promise<Doc[] | null> {
   try {
-    const response = await fetch(
-      'https://d3dg1063dc54p9.cloudfront.net/combined.json',
-    );
+    const apiHost =
+      import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+
+    const response = await fetch(apiHost + '/api/combine');
     const data = await response.json();
 
     const docs: Doc[] = [];
@@ -52,17 +54,13 @@ export function setLocalRecentDocs(doc: Doc): void {
     namePath = '.project';
   }
 
-  const docPath =
-    doc.name === 'default'
-      ? 'default'
-      : doc.language +
-        '/' +
-        namePath +
-        '/' +
-        doc.version +
-        '/' +
-        doc.model +
-        '/';
+  let docPath = 'default';
+  if (doc.location === 'local') {
+    docPath = 'local' + '/' + doc.name + '/';
+  } else if (doc.location === 'remote') {
+    docPath =
+      doc.language + '/' + namePath + '/' + doc.version + '/' + doc.model + '/';
+  }
   const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
   fetch(apiHost + '/api/docs_check', {
     method: 'POST',

frontend/src/store.ts

@@ -15,6 +15,7 @@ const store = configureStore({
       selectedDocs: doc !== null ? JSON.parse(doc) : null,
       sourceDocs: [
         {
+          location: '',
           language: '',
           name: 'default',
           version: '',

frontend/src/upload/Upload.tsx

@@ -0,0 +1,201 @@
+import React from 'react';
+import { useCallback, useEffect, useState } from 'react';
+import { useDropzone } from 'react-dropzone';
+import { useDispatch } from 'react-redux';
+import { ActiveState } from '../models/misc';
+import { getDocs } from '../preferences/preferenceApi';
+import { setSourceDocs } from '../preferences/preferenceSlice';
+
+export default function Upload({
+  modalState,
+  setModalState,
+}: {
+  modalState: ActiveState;
+  setModalState: (state: ActiveState) => void;
+}) {
+  const [docName, setDocName] = useState('');
+  const [files, setfiles] = useState<File[]>([]);
+  const [progress, setProgress] = useState<{
+    type: 'UPLOAD' | 'TRAINIING';
+    percentage: number;
+    taskId?: string;
+  }>();
+
+  function Progress({
+    title,
+    isCancellable = false,
+  }: {
+    title: string;
+    isCancellable?: boolean;
+  }) {
+    return (
+      <div className="mt-5 flex flex-col items-center gap-2">
+        <p className="text-xl tracking-[0.15px]">{title}...</p>
+        <p className="text-sm text-gray-2000">This may take several minutes</p>
+        <p className="mt-10 text-2xl">{progress?.percentage || 0}%</p>
+        <div className="mb-10 w-[50%]">
+          <div className="h-1 w-[100%] bg-blue-4000"></div>
+          <div
+            className={`relative bottom-1 h-1 bg-blue-5000 transition-all`}
+            style={{ width: `${progress?.percentage || 0}%` }}
+          ></div>
+        </div>
+        <button
+          onClick={() => {
+            setDocName('');
+            setfiles([]);
+            setProgress(undefined);
+            setModalState('INACTIVE');
+          }}
+          className={`rounded-md bg-blue-3000 px-4 py-2 text-sm font-medium text-white ${
+            isCancellable ? '' : 'hidden'
+          }`}
+        >
+          Finish
+        </button>
+      </div>
+    );
+  }
+
+  function UploadProgress() {
+    return <Progress title="Upload is in progress"></Progress>;
+  }
+
+  function TrainingProgress() {
+    const dispatch = useDispatch();
+    useEffect(() => {
+      (progress?.percentage ?? 0) < 100 &&
+        setTimeout(() => {
+          const apiHost = import.meta.env.VITE_API_HOST;
+          fetch(`${apiHost}/api/task_status?task_id=${progress?.taskId}`)
+            .then((data) => data.json())
+            .then((data) => {
+              if (data.status == 'SUCCESS') {
+                getDocs().then((data) => dispatch(setSourceDocs(data)));
+                setProgress(
+                  (progress) => progress && { ...progress, percentage: 100 },
+                );
+              } else {
+                setProgress(
+                  (progress) =>
+                    progress && {
+                      ...progress,
+                      percentage: data.result.current,
+                    },
+                );
+              }
+            });
+        }, 5000);
+    }, [progress, dispatch]);
+    return (
+      <Progress
+        title="Training is in progress"
+        isCancellable={progress?.percentage === 100}
+      ></Progress>
+    );
+  }
+
+  const onDrop = useCallback((acceptedFiles: File[]) => {
+    setfiles(acceptedFiles);
+    setDocName(acceptedFiles[0]?.name);
+  }, []);
+
+  const doNothing = () => undefined;
+
+  const uploadFile = () => {
+    const formData = new FormData();
+    files.forEach((file) => {
+      formData.append('file', file);
+    });
+    formData.append('name', docName);
+    formData.append('user', 'local');
+    const apiHost = import.meta.env.VITE_API_HOST;
+    const xhr = new XMLHttpRequest();
+    xhr.upload.addEventListener('progress', (event) => {
+      const progress = +((event.loaded / event.total) * 100).toFixed(2);
+      setProgress({ type: 'UPLOAD', percentage: progress });
+    });
+    xhr.onload = () => {
+      const { task_id } = JSON.parse(xhr.responseText);
+      setProgress({ type: 'TRAINIING', percentage: 0, taskId: task_id });
+    };
+    xhr.open('POST', `${apiHost + '/api/upload'}`);
+    xhr.send(formData);
+  };
+
+  const { getRootProps, getInputProps, isDragActive } = useDropzone({
+    onDrop,
+    multiple: true,
+    onDragEnter: doNothing,
+    onDragOver: doNothing,
+    onDragLeave: doNothing,
+  });
+
+  let view;
+  if (progress?.type === 'UPLOAD') {
+    view = <UploadProgress></UploadProgress>;
+  } else if (progress?.type === 'TRAINIING') {
+    view = <TrainingProgress></TrainingProgress>;
+  } else {
+    view = (
+      <>
+        <p className="mb-7 text-xl text-jet">Upload New Documentation</p>
+        <input
+          type="text"
+          className="h-10 w-[60%] rounded-md border-2 border-gray-5000 px-3 outline-none"
+          value={docName}
+          onChange={(e) => setDocName(e.target.value)}
+        ></input>
+        <div className="relative bottom-12 left-2 mt-[-18.39px]">
+          <span className="bg-white px-2 text-xs text-gray-4000">Name</span>
+        </div>
+        <div {...getRootProps()}>
+          <span className="rounded-md border border-blue-2000 px-4 py-2 font-medium text-blue-2000 hover:cursor-pointer">
+            <input type="button" {...getInputProps()} />
+            Choose Files
+          </span>
+        </div>
+        <div className="mt-9">
+          <p className="mb-5 font-medium text-eerie-black">Uploaded Files</p>
+          {files.map((file) => (
+            <p key={file.name} className="text-gray-6000">
+              {file.name}
+            </p>
+          ))}
+          {files.length === 0 && <p className="text-gray-6000">None</p>}
+        </div>
+        <div className="flex flex-row-reverse">
+          <button
+            onClick={uploadFile}
+            className="ml-6 rounded-md bg-blue-3000 py-2 px-6 text-white"
+          >
+            Train
+          </button>
+          <button
+            onClick={() => {
+              setDocName('');
+              setfiles([]);
+              setModalState('INACTIVE');
+            }}
+            className="font-medium"
+          >
+            Cancel
+          </button>
+        </div>
+      </>
+    );
+  }
+
+  return (
+    <article
+      className={`${
+        modalState === 'ACTIVE' ? 'visible' : 'hidden'
+      } absolute z-30  h-screen w-screen  bg-gray-alpha`}
+    >
+      <article className="mx-auto mt-24 flex w-[90vw] max-w-lg  flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
+        {view}
+      </article>
+    </article>
+  );
+}
+// TODO: sanitize all inputs

frontend/svg.d.ts

@@ -0,0 +1,7 @@
+declare module '*.svg' {
+  import * as React from 'react';
+
+  export const ReactComponent: React.FunctionComponent<
+    React.SVGProps<SVGSVGElement> & { title?: string }
+  >;
+}

frontend/tailwind.config.cjs

@@ -15,10 +15,17 @@ module.exports = {
         'gray-1000': '#F6F6F6',
         'gray-2000': 'rgba(0, 0, 0, 0.5)',
         'gray-3000': 'rgba(243, 243, 243, 1)',
+        'gray-4000': '#949494',
+        'gray-5000': '#BBBBBB',
+        'gray-6000': '#757575',
         'red-1000': 'rgb(254, 202, 202)',
         'red-2000': '#F44336',
         'red-3000': '#621B16',
         'blue-1000': '#7D54D1',
+        'blue-2000': '#002B49',
+        'blue-3000': '#4B02E2',
+        'blue-4000': 'rgba(0, 125, 255, 0.36)',
+        'blue-5000': 'rgba(0, 125, 255)',
       },
     },
   },

frontend/tsconfig.json

@@ -16,6 +16,6 @@
     "noEmit": true,
     "jsx": "react-jsx"
   },
-  "include": ["src"],
+  "include": ["src", "svg.d.ts"],
   "references": [{ "path": "./tsconfig.node.json" }]
 }

frontend/vite.config.ts

@@ -1,7 +1,8 @@
 import { defineConfig } from 'vite';
 import react from '@vitejs/plugin-react';
+import svgr from 'vite-plugin-svgr';
 
 // https://vitejs.dev/config/
 export default defineConfig({
-  plugins: [react()],
+  plugins: [react(), svgr()],
 });

scripts/ingest.py

@@ -3,14 +3,10 @@ import sys
 import nltk
 import dotenv
 import typer
-import ast
 
 from collections import defaultdict
-from pathlib import Path
 from typing import List, Optional
 
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-
 from parser.file.bulk import SimpleDirectoryReader
 from parser.schema.base import Document
 from parser.open_ai_func import call_openai_api, get_user_permission
@@ -18,6 +14,7 @@ from parser.py2doc import transform_to_docs
 from parser.py2doc import extract_functions_and_classes as extract_py
 from parser.js2doc import extract_functions_and_classes as extract_js
 from parser.java2doc import extract_functions_and_classes as extract_java
+from parser.token_func import group_split
 
 
 dotenv.load_dotenv()
@@ -38,14 +35,17 @@ def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
            file: Optional[List[str]] = typer.Option(None,
                                                    help="""File paths to use (Optional; overrides dir).
                                                         E.g. --file inputs/1.md --file inputs/2.md"""),
-           recursive: Optional[bool] = typer.Option(True,
-                                                   help="Whether to recursively search in subdirectories."),
-           limit: Optional[int] = typer.Option(None,
-                                                   help="Maximum number of files to read."),
+           recursive: Optional[bool] = typer.Option(True, help="Whether to recursively search in subdirectories."),
+           limit: Optional[int] = typer.Option(None, help="Maximum number of files to read."),
            formats: Optional[List[str]] = typer.Option([".rst", ".md"],
                                                    help="""List of required extensions (list with .)
-                                                        Currently supported: .rst, .md, .pdf, .docx, .csv, .epub, .html"""),
-           exclude: Optional[bool] = typer.Option(True, help="Whether to exclude hidden files (dotfiles).")):
+                                                        Currently supported: .rst, .md, .pdf, .docx, .csv, .epub, .html, .mdx"""),
+           exclude: Optional[bool] = typer.Option(True, help="Whether to exclude hidden files (dotfiles)."),
+           sample: Optional[bool] = typer.Option(False, help="Whether to output sample of the first 5 split documents."),
+           token_check: Optional[bool] = typer.Option(True, help="Whether to group small documents and split large."),
+           min_tokens: Optional[int] = typer.Option(150, help="Minimum number of tokens to not group."),
+           max_tokens: Optional[int] = typer.Option(2000, help="Maximum number of tokens to not split."),
+           ):
 
     """
         Creates index from specified location or files.
@@ -56,11 +56,21 @@ def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
         raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=file, recursive=recursive,
                                          required_exts=formats, num_files_limit=limit,
                                          exclude_hidden=exclude).load_data()
-        raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+
         # Here we split the documents, as needed, into smaller chunks.
         # We do this due to the context limits of the LLMs.
-        text_splitter = RecursiveCharacterTextSplitter()
-        docs = text_splitter.split_documents(raw_docs)
+        raw_docs = group_split(documents=raw_docs, min_tokens=min_tokens, max_tokens=max_tokens, token_check=token_check)
+        #Old method
+        # text_splitter = RecursiveCharacterTextSplitter()
+        # docs = text_splitter.split_documents(raw_docs)
+
+        #Sample feature
+        if sample == True:
+            for i in range(min(5, len(raw_docs))):
+                print(raw_docs[i].text)
+
+        docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+
 
         # Here we check for command line arguments for bot calls.
         # If no argument exists or the yes is not True, then the
@@ -109,3 +119,5 @@ def convert(dir: Optional[str] = typer.Option("inputs",
     transform_to_docs(functions_dict, classes_dict, formats, dir)
 if __name__ == "__main__":
   app()
+
+

scripts/parser/file/bulk.py

@@ -21,6 +21,7 @@ DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
     ".md": MarkdownParser(),
     ".rst": RstParser(),
     ".html": HTMLParser(),
+    ".mdx": MarkdownParser(),
 }
 
 
@@ -61,6 +62,7 @@ class SimpleDirectoryReader(BaseReader):
         file_extractor: Optional[Dict[str, BaseParser]] = None,
         num_files_limit: Optional[int] = None,
         file_metadata: Optional[Callable[[str], Dict]] = None,
+        chunk_size_max: int = 2048,
     ) -> None:
         """Initialize with parameters."""
         super().__init__()
@@ -74,6 +76,8 @@ class SimpleDirectoryReader(BaseReader):
         self.exclude_hidden = exclude_hidden
         self.required_exts = required_exts
         self.num_files_limit = num_files_limit
+        print("input_files")
+        print(input_files)
 
         if input_files:
             self.input_files = []

scripts/parser/file/markdown_parser.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 from parser.file.base_parser import BaseParser
+import tiktoken
 
 
 class MarkdownParser(BaseParser):
@@ -23,6 +24,7 @@ class MarkdownParser(BaseParser):
         *args: Any,
         remove_hyperlinks: bool = True,
         remove_images: bool = True,
+        max_tokens: int = 2048,
         # remove_tables: bool = True,
         **kwargs: Any,
     ) -> None:
@@ -30,8 +32,20 @@ class MarkdownParser(BaseParser):
         super().__init__(*args, **kwargs)
         self._remove_hyperlinks = remove_hyperlinks
         self._remove_images = remove_images
+        self._max_tokens = max_tokens
         # self._remove_tables = remove_tables
 
+
+    def tups_chunk_append(self, tups: List[Tuple[Optional[str], str]], current_header: Optional[str], current_text: str):
+        """Append to tups chunk."""
+        num_tokens = len(tiktoken.get_encoding("cl100k_base").encode(current_text))
+        if num_tokens > self._max_tokens:
+            chunks = [current_text[i:i + self._max_tokens] for i in range(0, len(current_text), self._max_tokens)]
+            for chunk in chunks:
+                tups.append((current_header, chunk))
+        else:
+            tups.append((current_header, current_text))
+        return tups
     def markdown_to_tups(self, markdown_text: str) -> List[Tuple[Optional[str], str]]:
         """Convert a markdown file to a dictionary.
 
@@ -50,13 +64,13 @@ class MarkdownParser(BaseParser):
                 if current_header is not None:
                     if current_text == "" or None:
                         continue
-                    markdown_tups.append((current_header, current_text))
+                    markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)
 
                 current_header = line
                 current_text = ""
             else:
                 current_text += line + "\n"
-        markdown_tups.append((current_header, current_text))
+        markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)
 
         if current_header is not None:
             # pass linting, assert keys are defined

scripts/parser/file/rst_parser.py

@@ -8,7 +8,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 from parser.file.base_parser import BaseParser
-
+import tiktoken
 
 class RstParser(BaseParser):
     """reStructuredText parser.
@@ -41,6 +41,7 @@ class RstParser(BaseParser):
         self._remove_whitespaces_excess = remove_whitespaces_excess
         self._remove_characters_excess = remove_characters_excess
 
+
     def rst_to_tups(self, rst_text: str) -> List[Tuple[Optional[str], str]]:
         """Convert a reStructuredText file to a dictionary.
 
@@ -68,6 +69,7 @@ class RstParser(BaseParser):
                 current_text = ""
             else:
                 current_text += line + "\n"
+
         rst_tups.append((current_header, current_text))
 
         #TODO: Format for rst

scripts/parser/open_ai_func.py

@@ -23,6 +23,7 @@ def num_tokens_from_string(string: str, encoding_name: str) -> int:
 @retry(tries=10, delay=60)
 def store_add_texts_with_retry(store, i):
     store.add_texts([i.page_content], metadatas=[i.metadata])
+    #store_pine.add_texts([i.page_content], metadatas=[i.metadata])
 
 def call_openai_api(docs, folder_name):
 # Function to create a vector store from the documents and save it to disk.
@@ -38,7 +39,13 @@ def call_openai_api(docs, folder_name):
     # cut first n docs if you want to restart
     #docs = docs[:n]
     c1 = 0
+    # pinecone.init(
+    #     api_key="",  # find at app.pinecone.io
+    #     environment="us-east1-gcp"  # next to api key in console
+    # )
+    #index_name = "pandas"
     store = FAISS.from_documents(docs_test, OpenAIEmbeddings())
+    #store_pine = Pinecone.from_documents(docs_test, OpenAIEmbeddings(), index_name=index_name)
 
     # Uncomment for MPNet embeddings
     # model_name = "sentence-transformers/all-mpnet-base-v2"

scripts/parser/token_func.py

@@ -0,0 +1,70 @@
+import re
+import tiktoken
+
+from typing import List
+from parser.schema.base import Document
+from math import ceil
+
+
+def separate_header_and_body(text):
+    header_pattern = r"^(.*?\n){3}"
+    match = re.match(header_pattern, text)
+    header = match.group(0)
+    body = text[len(header):]
+    return header, body
+
+def group_documents(documents: List[Document], min_tokens: int, max_tokens: int) -> List[Document]:
+    docs = []
+    current_group = None
+
+    for doc in documents:
+        doc_len = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
+
+        if current_group is None:
+            current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
+                                     extra_info=doc.extra_info)
+        elif len(tiktoken.get_encoding("cl100k_base").encode(current_group.text)) + doc_len < max_tokens and doc_len >= min_tokens:
+            current_group.text += " " + doc.text
+        else:
+            docs.append(current_group)
+            current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
+                                     extra_info=doc.extra_info)
+
+    if current_group is not None:
+        docs.append(current_group)
+
+    return docs
+
+def split_documents(documents: List[Document], max_tokens: int) -> List[Document]:
+    docs = []
+    for doc in documents:
+        token_length = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
+        if token_length <= max_tokens:
+            docs.append(doc)
+        else:
+            header, body = separate_header_and_body(doc.text)
+            num_body_parts = ceil(token_length / max_tokens)
+            part_length = ceil(len(body) / num_body_parts)
+            body_parts = [body[i:i + part_length] for i in range(0, len(body), part_length)]
+            for i, body_part in enumerate(body_parts):
+                new_doc = Document(text=header + body_part.strip(),
+                                   doc_id=f"{doc.doc_id}-{i}",
+                                   embedding=doc.embedding,
+                                   extra_info=doc.extra_info)
+                docs.append(new_doc)
+    return docs
+
+def group_split(documents: List[Document], max_tokens: int = 2000, min_tokens: int = 150, token_check: bool = True):
+    if token_check == False:
+        return documents
+    print("Grouping small documents")
+    try:
+        documents = group_documents(documents=documents, min_tokens=min_tokens, max_tokens=max_tokens)
+    except:
+        print("Grouping failed, try running without token_check")
+    print("Separating large documents")
+    try:
+        documents = split_documents(documents=documents, max_tokens=max_tokens)
+    except:
+        print("Grouping failed, try running without token_check")
+    return documents

scripts/requirements.txt

@@ -0,0 +1,131 @@
+aiodns==3.0.0
+aiohttp==3.8.3
+aiohttp-retry==2.8.3
+aiosignal==1.3.1
+alabaster==0.7.13
+aleph-alpha-client==2.16.0
+anyio==3.6.2
+argilla==1.3.0
+async-timeout==4.0.2
+attrs==22.2.0
+Babel==2.11.0
+backoff==2.2.1
+blobfile==2.0.1
+boto3==1.26.82
+botocore==1.29.82
+cffi==1.15.1
+charset-normalizer==2.1.1
+click==8.1.3
+cohere==3.4.0
+cryptography==39.0.2
+dataclasses-json==0.5.7
+decorator==5.1.1
+deeplake==3.2.12
+Deprecated==1.2.13
+dill==0.3.6
+docutils==0.19
+docx2txt==0.8
+ecdsa==0.18.0
+entrypoints==0.4
+escodegen==1.0.10
+esprima==4.0.1
+esutils==1.0.1
+et-xmlfile==1.1.0
+faiss-cpu==1.7.3
+filelock==3.9.0
+Flask==2.2.2
+frozenlist==1.3.3
+greenlet==2.0.2
+gunicorn==20.1.0
+h11==0.14.0
+httpcore==0.16.3
+httpx==0.23.3
+hub==3.0.1
+huggingface-hub==0.12.0
+humbug==0.2.8
+idna==3.4
+imagesize==1.4.1
+itsdangerous==2.1.2
+javalang==0.13.0
+Jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.2.0
+langchain==0.0.103
+lxml==4.9.2
+manifest-ml==0.1.1
+MarkupSafe==2.1.2
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+monotonic==1.6
+multidict==6.0.4
+multiprocess==0.70.14
+mypy-extensions==0.4.3
+nltk==3.8.1
+numcodecs==0.11.0
+numpy==1.23.5
+openai==0.27.0
+openpyxl==3.1.1
+packaging==23.0
+pandas==1.5.3
+pathos==0.3.0
+Pillow==9.4.0
+pox==0.3.2
+ppft==1.7.6.6
+py==1.11.0
+pyasn1==0.4.8
+pycares==4.3.0
+pycparser==2.21
+pycryptodomex==3.17
+pydantic==1.10.4
+Pygments==2.14.0
+PyJWT==2.6.0
+PyPDF2==3.0.1
+python-dateutil==2.8.2
+python-docx==0.8.11
+python-dotenv==0.21.1
+python-jose==3.3.0
+python-magic==0.4.27
+python-pptx==0.6.21
+pytz==2022.7.1
+PyYAML==6.0
+redis==4.5.1
+regex==2022.10.31
+requests==2.28.2
+retry==0.9.2
+rfc3986==1.5.0
+rsa==4.9
+s3transfer==0.6.0
+scikit-learn==1.2.1
+scipy==1.10.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.97
+six==1.16.0
+sniffio==1.3.0
+snowballstemmer==2.2.0
+Sphinx==6.1.3
+sphinxcontrib-applehelp==1.0.4
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.1
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.5
+SQLAlchemy==1.4.46
+sqlitedict==2.1.0
+tenacity==8.2.1
+threadpoolctl==3.1.0
+tiktoken==0.1.2
+tokenizers==0.13.2
+torch==1.13.1
+torchvision==0.14.1
+tqdm==4.64.1
+transformers==4.26.0
+typer==0.7.0
+typing-inspect==0.8.0
+typing_extensions==4.4.0
+unstructured==0.4.11
+urllib3==1.26.14
+Werkzeug==2.2.3
+wrapt==1.14.1
+XlsxWriter==3.0.8
+xxhash==3.2.0
+yarl==1.8.2