Merge branch 'main' of https://github.com/arc53/DocsGPT into main

.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: Build and push DocsGPT Docker image
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          
+      - name: Login to ghcr.io
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      # Runs a single command using the runners shell
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v2
+        with:
+          file: './application/Dockerfile'
+          platforms: linux/amd64
+          context: ./application
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt:latest

.gitignore

@@ -161,3 +161,4 @@ frontend/*.sw?
 
 application/vectors/
 
+**/inputs

CONTRIBUTING.md

@@ -0,0 +1,38 @@
+# Welcome to DocsGPT Contributing guideline
+
+Thank you for choosing this project to contribute to, we are all very grateful!
+
+# We accept different types of contributions
+
+📣 Discussions - where you can start a new topic or answer some questions
+
+🐞 Issues - Is how we track tasks, sometimes its bugs that need fixing, sometimes its new features
+
+🛠️ Pull requests - Is how you can suggest changes to our repository, to work on existing issue or to add new features
+
+📚 Wiki - where we have our documentation
+
+
+## 🐞 Issues and Pull requests
+
+We value contributions to our issues in form of discussion or suggestion, we recommend that you check out existing issues and our [Roadmap](https://github.com/orgs/arc53/projects/2)
+
+If you want to contribute by writing code there are few things that you should know before doing it:
+We have frontend (React, Vite) and Backend (python)
+
+### If you are looking to contribute to Frontend (⚛️React, Vite):
+Current frontend is being migrated from /application to /frontend with a new design, so please contribute to the new on. Check out this [Milestone](https://github.com/arc53/DocsGPT/milestone/1) and its issues also [Figma](https://www.figma.com/file/OXLtrl1EAy885to6S69554/DocsGPT?node-id=0%3A1&t=hjWVuxRg9yi5YkJ9-1)
+Please try to follow guidelines
+
+
+### If you are looking to contribute to Backend (🐍Python):
+Check out our issues, and contribute to /application or /scripts (ignore old  ingest_rst.py ingest_rst_sphinx.py files, they will be deprecated soon)
+Currently we don't have any tests(which would be useful😉) but before submitting you PR make sure that after you ingested some test data its queryable
+
+### Workflow:
+Create a fork, make changes on your forked repository, submit changes in a form of pull request
+
+## Questions / collaboration
+Please join our [Discord](https://discord.gg/n5BX8dh8rU) don't hesitate, we are very friendly and welcoming to new contributors.
+
+# Thank you so much for considering to contribute to DocsGPT!🙏

README.md

@@ -57,9 +57,11 @@ Copy .env_sample and create .env with your openai api token
 
 ## [Guides](https://github.com/arc53/docsgpt/wiki)
 
-
+## [Interested in contributing?](https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md)
 
 ## [How to use any other documentation](https://github.com/arc53/docsgpt/wiki/How-to-train-on-other-documentation)
 
+## [How to host it locally (so all data will stay on-premises)](https://github.com/arc53/DocsGPT/wiki/How-to-use-different-LLM's#hosting-everything-locally)
+
 Built with [🦜️🔗 LangChain](https://github.com/hwchase17/langchain)
 

application/app.py

@@ -1,54 +1,88 @@
 import os
-import pickle
+import json
+
 import dotenv
-import datetime
-from flask import Flask, request, render_template
-# os.environ["LANGCHAIN_HANDLER"] = "langchain"
-import faiss
-from langchain import OpenAI
-from langchain.chains import VectorDBQAWithSourcesChain
-from langchain.prompts import PromptTemplate
 import requests
+from flask import Flask, request, render_template
+from langchain import FAISS
+from langchain import OpenAI, VectorDBQA, HuggingFaceHub, Cohere
+from langchain.chains.question_answering import load_qa_chain
+from langchain.embeddings import OpenAIEmbeddings, HuggingFaceHubEmbeddings, CohereEmbeddings, HuggingFaceInstructEmbeddings
+from langchain.prompts import PromptTemplate
+
+# os.environ["LANGCHAIN_HANDLER"] = "langchain"
+
+if os.getenv("LLM_NAME") is not None:
+    llm_choice = os.getenv("LLM_NAME")
+else:
+    llm_choice = "openai"
+
+if os.getenv("EMBEDDINGS_NAME") is not None:
+    embeddings_choice = os.getenv("EMBEDDINGS_NAME")
+else:
+    embeddings_choice = "openai_text-embedding-ada-002"
+
+
+
+if llm_choice == "manifest":
+    from manifest import Manifest
+    from langchain.llms.manifest import ManifestWrapper
+
+    manifest = Manifest(
+        client_name="huggingface",
+        client_connection="http://127.0.0.1:5000"
+    )
 
 # Redirect PosixPath to WindowsPath on Windows
 import platform
+
 if platform.system() == "Windows":
     import pathlib
+
     temp = pathlib.PosixPath
     pathlib.PosixPath = pathlib.WindowsPath
 
 # loading the .env file
 dotenv.load_dotenv()
 
-
 with open("combine_prompt.txt", "r") as f:
     template = f.read()
 
-# check if OPENAI_API_KEY is set
-if os.getenv("OPENAI_API_KEY") is not None:
-    api_key_set = True
+with open("combine_prompt_hist.txt", "r") as f:
+    template_hist = f.read()
 
+if os.getenv("API_KEY") is not None:
+    api_key_set = True
 else:
     api_key_set = False
-
-
+if os.getenv("EMBEDDINGS_KEY") is not None:
+    embeddings_key_set = True
+else:
+    embeddings_key_set = False
 
 app = Flask(__name__)
 
 
 @app.route("/")
 def home():
-    return render_template("index.html", api_key_set=api_key_set)
+    return render_template("index.html", api_key_set=api_key_set, llm_choice=llm_choice,
+                           embeddings_choice=embeddings_choice)
 
 
 @app.route("/api/answer", methods=["POST"])
 def api_answer():
     data = request.get_json()
     question = data["question"]
+    history = data["history"]
     if not api_key_set:
         api_key = data["api_key"]
     else:
-        api_key = os.getenv("OPENAI_API_KEY")
+        api_key = os.getenv("API_KEY")
+    if not embeddings_key_set:
+        embeddings_key = data["embeddings_key"]
+    else:
+        embeddings_key = os.getenv("EMBEDDINGS_KEY")
+
 
     # check if the vectorstore is set
     if "active_docs" in data:
@@ -59,21 +93,44 @@ def api_answer():
         vectorstore = ""
 
     # loading the index and the store and the prompt template
-    index = faiss.read_index(f"{vectorstore}docs.index")
+    # Note if you have used other embeddings than OpenAI, you need to change the embeddings
+    if embeddings_choice == "openai_text-embedding-ada-002":
+        docsearch = FAISS.load_local(vectorstore, OpenAIEmbeddings(openai_api_key=embeddings_key))
+    elif embeddings_choice == "huggingface_sentence-transformers/all-mpnet-base-v2":
+        docsearch = FAISS.load_local(vectorstore, HuggingFaceHubEmbeddings())
+    elif embeddings_choice == "huggingface_hkunlp/instructor-large":
+        docsearch = FAISS.load_local(vectorstore, HuggingFaceInstructEmbeddings())
+    elif embeddings_choice == "cohere_medium":
+        docsearch = FAISS.load_local(vectorstore, CohereEmbeddings(cohere_api_key=embeddings_key))
 
-    with open(f"{vectorstore}faiss_store.pkl", "rb") as f:
-        store = pickle.load(f)
-
-    store.index = index
     # create a prompt template
-    c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template)
-    # create a chain with the prompt template and the store
+    if history:
+        history = json.loads(history)
+        template_temp = template_hist.replace("{historyquestion}", history[0]).replace("{historyanswer}", history[1])
+        c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template_temp, template_format="jinja2")
+    else:
+        c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template, template_format="jinja2")
+
+    if llm_choice == "openai":
+        llm = OpenAI(openai_api_key=api_key, temperature=0)
+    elif llm_choice == "manifest":
+        llm = ManifestWrapper(client=manifest, llm_kwargs={"temperature": 0.001, "max_tokens": 2048})
+    elif llm_choice == "huggingface":
+        llm = HuggingFaceHub(repo_id="bigscience/bloom", huggingfacehub_api_token=api_key)
+    elif llm_choice == "cohere":
+        llm = Cohere(model="command-xlarge-nightly", cohere_api_key=api_key)
+
+    qa_chain = load_qa_chain(llm=llm, chain_type="map_reduce",
+                             combine_prompt=c_prompt)
+
+    chain = VectorDBQA(combine_documents_chain=qa_chain, vectorstore=docsearch, k=4)
 
-    chain = VectorDBQAWithSourcesChain.from_llm(llm=OpenAI(openai_api_key=api_key, temperature=0), vectorstore=store, combine_prompt=c_prompt)
     # fetch the answer
-    result = chain({"question": question})
+    result = chain({"query": question})
+    print(result)
 
     # some formatting for the frontend
+    result['answer'] = result['result']
     result['answer'] = result['answer'].replace("\\n", "<br>")
     result['answer'] = result['answer'].replace("SOURCES:", "")
     # mock result
@@ -83,30 +140,33 @@ def api_answer():
     # }
     return result
 
+
 @app.route("/api/docs_check", methods=["POST"])
 def check_docs():
     # check if docs exist in a vectorstore folder
     data = request.get_json()
     vectorstore = "vectors/" + data["docs"]
     base_path = 'https://raw.githubusercontent.com/arc53/DocsHUB/main/'
-    #
-    if os.path.exists(vectorstore):
+    if os.path.exists(vectorstore) or data["docs"] == "default":
         return {"status": 'exists'}
     else:
-        r = requests.get(base_path + vectorstore + "docs.index")
-        # save to vectors directory
-        # check if the directory exists
-        if not os.path.exists(vectorstore):
-            os.makedirs(vectorstore)
+        r = requests.get(base_path + vectorstore + "index.faiss")
 
-        with open(vectorstore + "docs.index", "wb") as f:
-            f.write(r.content)
-        # download the store
-        r = requests.get(base_path + vectorstore + "faiss_store.pkl")
-        with open(vectorstore + "faiss_store.pkl", "wb") as f:
-            f.write(r.content)
+        if r.status_code != 200:
+            return {"status": 'null'}
+        else:
+            if not os.path.exists(vectorstore):
+                os.makedirs(vectorstore)
+            with open(vectorstore + "index.faiss", "wb") as f:
+                f.write(r.content)
+
+            # download the store
+            r = requests.get(base_path + vectorstore + "index.pkl")
+            with open(vectorstore + "index.pkl", "wb") as f:
+                f.write(r.content)
+
+        return {"status": 'loaded'}
 
-        return {"status":  'loaded'}
 
 # handling CORS
 @app.after_request

application/combine_prompt.txt

@@ -20,8 +20,8 @@ Source: 0-pl
 FINAL ANSWER: You can't eat vegetables using pandas. You can only eat them using your mouth.
 SOURCES:
 
-QUESTION: {question}
+QUESTION: {{ question }}
 =========
-{summaries}
+{{ summaries }}
 =========
 FINAL ANSWER:

application/combine_prompt_hist.txt

@@ -0,0 +1,27 @@
+You are a DocsGPT bot assistant by Arc53 that provides help with programming libraries. You give thorough answers with code examples.
+Given the following extracted parts of a long document and a question, create a final answer with references ("SOURCES").
+ALWAYS return a "SOURCES" part in your answer. You can also remeber things from previous questions and use them in your answer.
+
+QUESTION: How to merge tables in pandas?
+=========
+Content: pandas provides various facilities for easily combining together Series or DataFrame with various kinds of set logic for the indexes and relational algebra functionality in the case of join / merge-type operations.
+Source: 28-pl
+Content: pandas provides a single function, merge(), as the entry point for all standard database join operations between DataFrame or named Series objects: \n\npandas.merge(left, right, how='inner', on=None, left_on=None, right_on=None, left_index=False, right_index=False, sort=False, suffixes=('_x', '_y'), copy=True, indicator=False, validate=None)
+Source: 30-pl
+=========
+FINAL ANSWER: To merge two tables in pandas, you can use the pd.merge() function. The basic syntax is: \n\npd.merge(left, right, on, how) \n\nwhere left and right are the two tables to merge, on is the column to merge on, and how is the type of merge to perform. \n\nFor example, to merge the two tables df1 and df2 on the column 'id', you can use: \n\npd.merge(df1, df2, on='id', how='inner')
+SOURCES: 28-pl 30-pl
+
+QUESTION: {{ historyquestion }}
+=========
+CONTENT:
+SOURCE:
+=========
+FINAL ANSWER: {{ historyanswer }}
+SOURCES:
+
+QUESTION: {{ question }}
+=========
+{{ summaries }}
+=========
+FINAL ANSWER:

application/requirements.txt

@@ -45,6 +45,7 @@ pytz==2022.7.1
 PyYAML==6.0
 regex==2022.10.31
 requests==2.28.2
+retry==0.9.2
 six==1.16.0
 snowballstemmer==2.2.0
 Sphinx==6.1.3
@@ -60,10 +61,11 @@ tiktoken==0.1.2
 tokenizers==0.13.2
 tqdm==4.64.1
 transformers==4.26.0
+typer==0.7.0
 typing-inspect==0.8.0
 typing_extensions==4.4.0
 unstructured==0.4.8
 urllib3==1.26.14
-Werkzeug==2.2.2
+Werkzeug==2.2.3
 XlsxWriter==3.0.8
 yarl==1.8.2

application/static/src/chat.js

@@ -25,6 +25,8 @@ if (el) {
 
             body: JSON.stringify({question: message,
                 api_key: localStorage.getItem('apiKey'),
+                embeddings_key: localStorage.getItem('apiKey'),
+                history: localStorage.getItem('chatHistory'),
                 active_docs: localStorage.getItem('activeDocs')}),
         })
             .then(response => response.json())
@@ -38,9 +40,12 @@ if (el) {
                 chatWindow.scrollTop = chatWindow.scrollHeight;
                 document.getElementById("button-submit").innerHTML = 'Send';
                 document.getElementById("button-submit").disabled = false;
+                let chatHistory = [message, data.answer];
+                localStorage.setItem('chatHistory', JSON.stringify(chatHistory));
             })
             .catch((error) => {
                 console.error('Error:', error);
+                console.log(error);
                 document.getElementById("button-submit").innerHTML = 'Send';
                 document.getElementById("button-submit").disabled = false;
             });

application/templates/index.html

@@ -131,13 +131,17 @@ This will return a new DataFrame with all the columns from both tables, and only
                     var option = document.createElement("option");
                     if (docsIndex[key].name == docsIndex[key].language) {
                         option.text = docsIndex[key].name + " " + docsIndex[key].version;
-                        option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/";
-                        select.add(option);
+                        option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
+                        if (docsIndex[key].model == "{{ embeddings_choice }}") {
+                            select.add(option);
+                        }
                     }
                     else {
                         option.text = docsIndex[key].name + " " + docsIndex[key].version;
-                        option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/";
-                        select.add(option);
+                        option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
+                        if (docsIndex[key].model == "{{ embeddings_choice }}") {
+                            select.add(option);
+                        }
                     }
                 }
 

frontend/.eslintrc.cjs

@@ -19,6 +19,12 @@ module.exports = {
   plugins: ['react'],
   rules: {
     'react/react-in-jsx-scope': 'off',
+    'prettier/prettier': [
+      'error',
+      {
+        endOfLine: 'auto',
+      },
+    ],
   },
   settings: {
     'import/parsers': {
@@ -34,10 +40,4 @@ module.exports = {
       },
     },
   },
-  'prettier/prettier': [
-    'error',
-    {
-      endOfLine: 'auto',
-    },
-  ],
-}
+};

frontend/package.json

@@ -19,8 +19,10 @@
     ]
   },
   "dependencies": {
+    "@reduxjs/toolkit": "^1.9.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "react-redux": "^8.0.5",
     "react-router-dom": "^6.8.1"
   },
   "devDependencies": {

frontend/src/About.tsx

@@ -0,0 +1,54 @@
+//TODO - Add hyperlinks to text
+//TODO - Styling
+
+export default function About() {
+  return (
+    //Parent div for all content shown through App.tsx routing needs to have this styling. Might change when state management is updated.
+    <div className="mx-6 grid min-h-screen">
+      <article className=" mx-auto my-auto flex w-full max-w-6xl flex-col place-items-center gap-6 rounded-lg bg-gray-100 p-6 text-jet lg:p-10 xl:p-16">
+        <p className="text-3xl font-semibold">About DocsGPT 🦖</p>
+        <p className="mt-4 text-xl font-bold">
+          Find the information in your documentation through AI-powered
+          open-source chatbot. Powered by GPT-3, Faiss and LangChain.
+        </p>
+
+        <div>
+          <p className="text-lg">
+            If you want to add your own documentation, please follow the
+            instruction below:
+          </p>
+          <p className="mt-4 text-lg">
+            1. Navigate to{' '}
+            <span className="bg-gray-200 italic"> /application</span> folder
+          </p>
+          <p className="mt-4 text-lg">
+            2. Install dependencies from{' '}
+            <span className="bg-gray-200 italic">
+              pip install -r requirements.txt
+            </span>
+          </p>
+          <p className="mt-4 text-lg">
+            3. Prepare a <span className="bg-gray-200 italic">.env</span> file.
+            Copy <span className="bg-gray-200 italic">.env_sample</span> and
+            create <span className="bg-gray-200 italic">.env</span> with your
+            OpenAI API token
+          </p>
+          <p className="mt-4 text-lg">
+            4. Run the app with{' '}
+            <span className="bg-gray-200 italic">python app.py</span>
+          </p>
+        </div>
+
+        <p className="text-lg">
+          Currently It uses python pandas documentation, so it will respond to
+          information relevant to pandas. If you want to train it on different
+          documentation - please follow this guide.
+        </p>
+
+        <p className="mt-4 text-lg">
+          If you want to launch it on your own server - follow this guide.
+        </p>
+      </article>
+    </div>
+  );
+}

frontend/src/App.css

@@ -1,4 +0,0 @@
-html,
-body {
-  min-height: 100vh;
-}

frontend/src/App.tsx

@@ -1,55 +1,29 @@
-import { useEffect, useState } from 'react';
 import { Routes, Route } from 'react-router-dom';
-import Navigation from './components/Navigation/Navigation';
-import DocsGPT from './components/DocsGPT';
-import APIKeyModal from './components/APIKeyModal';
-import './App.css';
+import Navigation from './Navigation';
+import Conversation from './conversation/Conversation';
+import About from './About';
+import { useState } from 'react';
+import { ActiveState } from './models/misc';
 
 export default function App() {
-  //Currently using primitive state management. Will most likely be replaced with Redux.
-  const [isMobile, setIsMobile] = useState(true);
-  const [isMenuOpen, setIsMenuOpen] = useState(true);
-  const [isApiModalOpen, setIsApiModalOpen] = useState(true);
-  const [apiKey, setApiKey] = useState('');
-
-  const handleResize = () => {
-    if (window.innerWidth > 768 && isMobile) {
-      setIsMobile(false);
-    } else {
-      setIsMobile(true);
-    }
-  };
-
-  useEffect(() => {
-    window.addEventListener('resize', handleResize);
-    handleResize();
-
-    return () => {
-      window.removeEventListener('resize', handleResize);
-    };
-  }, []);
+  const [navState, setNavState] = useState<ActiveState>('ACTIVE');
 
   return (
-    <div
-      className={`${
-        isMobile ? 'flex-col' : 'flex-row'
-      } relative flex transition-all`}
-    >
-      <APIKeyModal
-        apiKey={apiKey}
-        setApiKey={setApiKey}
-        isApiModalOpen={isApiModalOpen}
-        setIsApiModalOpen={setIsApiModalOpen}
-      />
+    <div className="min-h-full min-w-full">
       <Navigation
-        isMobile={isMobile}
-        isMenuOpen={isMenuOpen}
-        setIsMenuOpen={setIsMenuOpen}
-        setIsApiModalOpen={setIsApiModalOpen}
+        navState={navState}
+        setNavState={(val: ActiveState) => setNavState(val)}
       />
-      <Routes>
-        <Route path="/" element={<DocsGPT isMenuOpen={isMenuOpen} />} />
-      </Routes>
+      <div
+        className={`transition-all duration-200 ${
+          navState === 'ACTIVE' ? 'ml-0 md:ml-72 lg:ml-96' : ' ml-0 md:ml-16'
+        }`}
+      >
+        <Routes>
+          <Route path="/" element={<Conversation />} />
+          <Route path="/about" element={<About />} />
+        </Routes>
+      </div>
     </div>
   );
 }

frontend/src/Avatar.tsx

@@ -0,0 +1,9 @@
+export default function Avatar({
+  avatar,
+  size,
+}: {
+  avatar: string;
+  size?: 'SMALL' | 'MEDIUM' | 'LARGE';
+}) {
+  return <div>{avatar}</div>;
+}

frontend/src/Hero.tsx

@@ -0,0 +1,21 @@
+export default function Hero({ className = '' }: { className?: string }) {
+  return (
+    <div className={`flex flex-col ${className}`}>
+      <p className="mb-10 text-center text-4xl font-semibold">
+        DocsGPT <span className="text-3xl">🦖</span>
+      </p>
+      <p className="mb-3 text-center">
+        Welcome to DocsGPT, your technical documentation assistant!
+      </p>
+      <p className="mb-3 text-center">
+        Enter a query related to the information in the documentation you
+        selected to receive and we will provide you with the most relevant
+        answers.
+      </p>
+      <p className="text-center">
+        Start by entering your query in the input field below and we will do the
+        rest!
+      </p>
+    </div>
+  );
+}

frontend/src/Navigation.tsx

@@ -0,0 +1,123 @@
+import { NavLink } from 'react-router-dom';
+import Arrow1 from './assets/arrow.svg';
+import Hamburger from './assets/hamburger.svg';
+import Key from './assets/key.svg';
+import Info from './assets/info.svg';
+import Link from './assets/link.svg';
+import { ActiveState } from './models/misc';
+import APIKeyModal from './preferences/APIKeyModal';
+import SelectDocsModal from './preferences/SelectDocsModal';
+import { useSelector } from 'react-redux';
+import {
+  selectApiKeyStatus,
+  selectSelectedDocsStatus,
+} from './preferences/preferenceSlice';
+import { useState } from 'react';
+
+//TODO - Need to replace Chat button to open secondary nav with scrollable past chats option and new chat at top
+//TODO - Need to add Discord and Github links
+
+export default function Navigation({
+  navState,
+  setNavState,
+}: {
+  navState: ActiveState;
+  setNavState: (val: ActiveState) => void;
+}) {
+  const isApiKeySet = useSelector(selectApiKeyStatus);
+  const [apiKeyModalState, setApiKeyModalState] = useState<ActiveState>(
+    isApiKeySet ? 'INACTIVE' : 'ACTIVE',
+  );
+
+  const isSelectedDocsSet = useSelector(selectSelectedDocsStatus);
+  const [selectedDocsModalState, setSelectedDocsModalState] =
+    useState<ActiveState>(isSelectedDocsSet ? 'INACTIVE' : 'ACTIVE');
+
+  return (
+    <>
+      <div
+        className={`${
+          navState === 'INACTIVE' && '-ml-96 md:-ml-[14rem] lg:-ml-80'
+        } fixed z-10 flex h-full w-72 flex-col border-r-2 border-gray-100 bg-gray-50 transition-all duration-200 lg:w-96`}
+      >
+        <div className={'h-16 w-full border-b-2 border-gray-100'}>
+          <button
+            className="float-right mr-5 mt-5 h-5 w-5"
+            onClick={() =>
+              setNavState(navState === 'ACTIVE' ? 'INACTIVE' : 'ACTIVE')
+            }
+          >
+            <img
+              src={Arrow1}
+              alt="menu toggle"
+              className={`${
+                navState === 'INACTIVE' ? 'rotate-180' : 'rotate-0'
+              }  m-auto w-3 transition-all duration-200`}
+            />
+          </button>
+        </div>
+        <div className="flex-grow border-b-2 border-gray-100"></div>
+
+        <div className="flex flex-col gap-2 border-b-2 border-gray-100 py-2">
+          <div
+            className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
+            onClick={() => {
+              setApiKeyModalState('ACTIVE');
+            }}
+          >
+            <img src={Key} alt="key" className="ml-2 w-6" />
+            <p className="my-auto text-eerie-black">Reset Key</p>
+          </div>
+
+          <div
+            className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
+            onClick={() => {
+              setSelectedDocsModalState('ACTIVE');
+            }}
+          >
+            <img src={Link} alt="key" className="ml-2 w-5" />
+            <p className="my-auto text-eerie-black">
+              Select Source Documentation
+            </p>
+          </div>
+        </div>
+
+        <div className="flex h-48 flex-col border-b-2 border-gray-100">
+          <NavLink
+            to="/about"
+            className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
+          >
+            <img src={Info} alt="info" className="ml-2 w-5" />
+            <p className="my-auto text-eerie-black">About</p>
+          </NavLink>
+
+          <div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
+            <img src={Link} alt="link" className="ml-2 w-5" />
+            <p className="my-auto text-eerie-black">Discord</p>
+          </div>
+
+          <div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
+            <img src={Link} alt="link" className="ml-2 w-5" />
+            <p className="my-auto text-eerie-black">Github</p>
+          </div>
+        </div>
+      </div>
+      <button
+        className="fixed mt-5 ml-6 h-6 w-6 md:hidden"
+        onClick={() => setNavState('ACTIVE')}
+      >
+        <img src={Hamburger} alt="menu toggle" className="w-7" />
+      </button>
+      <SelectDocsModal
+        modalState={selectedDocsModalState}
+        setModalState={setSelectedDocsModalState}
+        isCancellable={isSelectedDocsSet}
+      />
+      <APIKeyModal
+        modalState={apiKeyModalState}
+        setModalState={setApiKeyModalState}
+        isCancellable={isApiKeySet}
+      />
+    </>
+  );
+}

frontend/src/assets/exit.svg

@@ -0,0 +1,10 @@
+<svg width="800" height="800" viewBox="0 0 800 800" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_202_7)">
+<path d="M750 50L50 750M750 750L50 50" stroke="black" stroke-opacity="0.54" stroke-width="100" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_202_7">
+<rect width="800" height="800" fill="white"/>
+</clipPath>
+</defs>
+</svg>

frontend/src/assets/hamburger.svg

@@ -0,0 +1,3 @@
+<svg width="600" height="450" viewBox="0 0 600 450" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M25 25H575M25 225H575M25 425H575" stroke="black" stroke-opacity="0.54" stroke-width="50" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

frontend/src/assets/send.svg

@@ -0,0 +1,3 @@
+<svg width="21" height="18" viewBox="0 0 21 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0.00999999 18L21 9L0.00999999 0L0 7L15 9L0 11L0.00999999 18Z" fill="black" fill-opacity="0.54"/>
+</svg>

frontend/src/assets/spinner.svg

@@ -0,0 +1,9 @@
+<svg width="30" height="33" viewBox="0 0 30 33" fill="none" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<rect width="30" height="33" fill="url(#pattern0)"/>
+<defs>
+<pattern id="pattern0" patternContentUnits="objectBoundingBox" width="1" height="1">
+<use xlink:href="#image0_1_917" transform="scale(0.0166667 0.0151515)"/>
+</pattern>
+<image id="image0_1_917" width="60" height="66" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADwAAABCCAYAAAAL1LXDAAAGuElEQVRoge2aSXPaSheG3wYJBJiSQAx2mCInhqLiSjZJKllml1+cn5BVUnE5k4sEDAZjZgkHMUiIvgtf9FkxYIwxn+H63bXULZ1H3eo+ffoQSinFf0i2/7cBq9YD8KbrAXjT9QC86WJW9SJKKTqdDlRVhc1mg9frhcvlQq/XA8MwYFl2JXbcGbAsy1AUBZVKBaqqQtd1MAwDTdNACIGmaWAYBjzPY2trC06nE6FQCD6fD5RS2Gw22GzLH4BkGZ7WaDRCu92GqqpotVqoVqtQFAW6rkPXdQAXPfz3qwgh5jWO4+B0OuH1ehGNRuH1euHz+eDxeG5rnvWdywCu1Wr49esXFEVBp9NBr9e7Aje3Qf9+BI/Hg1gshng8jkgkAkopCCG3NfV2wM1mE1+/fkWpVDJ7cpY4joPL5QKlFL1eD4PBACzLglKK4XB41ThCwPM8kskktre3wfM87Hb7rcBvDGwYBlqtFsrlMkqlEprNJgzDmAgnCAICgQAEQTAnJZZlYRiG+T+rqmqODEVRJo4OlmXh9Xqxv7+PaDQKp9O5OmBZlvH582dUKpUrver1ehEMBuHxeOD3+81/0GazgRACu91uqT8ajcyJqdlsQtd1yLKMTCZjTnSj0cisHwwG8fjxY8RiMfA8vxrgDx8+oFarWXqBYRiIoghJkrC9vQ2PxwOHw7GQQYZhoNFooFQqodFooF6vWz6s2+1GJBJBKpVCIBC48fCeG/jPnz/49OkTTk5OLF+d4zjs7u4imUyC5/mlLSWGYUBRFOTzeWQyGfT7ffMewzDw+Xx49+7djWfxuYBbrRZ+/PiBfD5vfm2Hw4FEIoF0Og1RFG+IM780TUOhUMDBwQE6nY7lniRJePXqFdxu99w9fW139Pt9ZLNZFItFE5ZlWUSjUaTTafh8vgUw5pfD4YAkSXj9+jUCgQAY5n++Uj6fRzabhaqqcz/vWuCzszPkcjlzSBFCIIoiUqkURFG8E2/obzEMg3g8jhcvXiAcDpuTH6UUP3/+xPHx8cRlbeKzZt0cDAY4PDxEt9s1r1FK8fLlSwQCgVsg3FyEEMTjcXNZq1QqAIBut4vj42OEQiGEw+FrnzOzew4ODtBut80yz/N4//49RFFciteziILBIPb398FxnHlNlmVks9m52k8FrtfrqFarFqciFoutbBhPE8MwiMVi2N3dNf/n0WiEYrGIer1uWUEmaaLlg8EAhUIBsiybPenz+RCJRBZeX5etRCIBQRBM+8ZD+7oJbCKw3W5HNBqFJEngeR4ulwuSJCEYDC7f8gUliiISiYSlAyqVChqNxsx2EycthmEQDofh9/vR7XZxdHSEp0+frmyTPo9YlsXOzg7Ozs5QLpcBAJ1OB7VaDZIkTW039WckhMDhcEAQBLx582bp+9JlSBAERKNRs6xpGhRFmdlmrWNaLMsiGAzC5XIBuFgyz8/PIcvy1DZrDQxcQF/eOXW7XdRqtan11x7Y4/FYnKDRaIRmszm1/toD22w2CIJgKV/2DK/UX4VRdylCiGX1oJRC1/WpMbW1Bx5vJMauJqUUDMNc2UqOtfbAwMXENZ6pAZixsknaCODhcGjx+QkhU/39jQC22Ww4Pz83y4ZhIBQKTa67KqPuSpRSy1aREIJerze1/toDE0LQ6XTMTQSlFOFweCr02gMDQLlcvhJJ3dhlqd1uo9/vmzEth8MBp9MJt9s9sf7aA6uqitPTU7PMcRx2dnam1l9rYMMwIMuyuSQRQsBx3MwA41oD9/t95HI5s0wpRSwWm3lUu7bAuq7j9+/flvV3fJg3K8i4tsCtVgsnJyfQNA3AxXBOJBLw+/2bBzwajVCtVq/EzB89enTt2fHKsniWpeFwiFwuh+/fv5tnXeM9cSQSubb9WgFTSiHLMj5+/GiZmARBwNu3b+d6xloBV6tVfPnyxXLN7Xbj+fPnFn96ltYCWNM0NBoNHB0dodlsmr3rdruxt7c309H4W2sBfHp6im/fvlmcDIZhEIlE8OTJk7l7F7jHwONkt0wmYzmfHsvn8yGVSt04ueXeAY/DM8Vi0UyLunzYTQjBs2fPkE6nFzoNuRfAlFIoigJZltFqtaCqqpnsdnk2drvdSKfTSCaTNxrGl3UvgIfDIQ4PD1Eul68MXeAiMikIAiRJgiRJt0pMuxfAhmHAMIyJsJcTaERRtCS1LKJ7ATz2lAqFAgghcLlc2NragiAI2NvbmxqQW0T3Anh8Hs2yLMLhsHk27fF4Fk4xnKalpA8vQ5RStFqtO01yA+4R8Kq0ltvD2+gBeNP1ALzp+s8B/wPYvPrTcSgesQAAAABJRU5ErkJggg=="/>
+</defs>
+</svg>

frontend/src/components/APIKeyModal.tsx

@@ -1,60 +0,0 @@
-import { useState } from 'react';
-
-export default function APIKeyModal({
-  isApiModalOpen,
-  setIsApiModalOpen,
-  apiKey,
-  setApiKey,
-}: {
-  isApiModalOpen: boolean;
-  setIsApiModalOpen: React.Dispatch<React.SetStateAction<boolean>>;
-  apiKey: string;
-  setApiKey: React.Dispatch<React.SetStateAction<string>>;
-}) {
-  const [formError, setFormError] = useState(false);
-
-  const handleResetKey = () => {
-    if (!apiKey) {
-      setFormError(true);
-    } else {
-      setFormError(false);
-      setIsApiModalOpen(false);
-    }
-  };
-
-  return (
-    <div
-      className={`${
-        isApiModalOpen ? 'visible' : 'hidden'
-      } absolute z-30  h-screen w-screen  bg-gray-alpha`}
-    >
-      <div className="mx-auto mt-24 flex w-128 flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
-        <p className="text-xl text-jet">OpenAI API Key</p>
-        <p className="text-lg leading-5 text-gray-500">
-          Before you can start using DocsGPT we need you to provide an API key
-          for llm. Currently, we support only OpenAI but soon many more. You can
-          find it here.
-        </p>
-        <input
-          type="text"
-          className="h-10 w-full border-b-2 border-jet focus:outline-none"
-          value={apiKey}
-          maxLength={100}
-          placeholder="API Key"
-          onChange={(e) => setApiKey(e.target.value)}
-        />
-        <div className="flex justify-between">
-          {formError && (
-            <p className="text-sm text-red-500">Please enter a valid API key</p>
-          )}
-          <button
-            onClick={handleResetKey}
-            className="ml-auto h-10 w-20 rounded-lg bg-violet-800 text-white transition-all hover:bg-violet-700"
-          >
-            Save
-          </button>
-        </div>
-      </div>
-    </div>
-  );
-}

frontend/src/components/DocsGPT.tsx

@@ -1,7 +0,0 @@
-export default function DocsGPT({ isMenuOpen }: { isMenuOpen: boolean }) {
-  return (
-    <div className={`${isMenuOpen ? 'md:ml-72 lg:ml-96' : 'ml-16'}`}>
-      Docs GPT Chat Placeholder
-    </div>
-  );
-}

frontend/src/components/Navigation/Navigation.tsx

@@ -1,103 +0,0 @@
-import React, { useState } from 'react';
-import Arrow1 from './imgs/arrow.svg';
-import Key from './imgs/key.svg';
-import Info from './imgs/info.svg';
-import Link from './imgs/link.svg';
-
-function MobileNavigation() {
-  return <div>Mobile Navigation</div>;
-}
-
-function DesktopNavigation({
-  isMenuOpen,
-  setIsMenuOpen,
-  setIsApiModalOpen,
-}: {
-  isMenuOpen: boolean;
-  setIsMenuOpen: React.Dispatch<React.SetStateAction<boolean>>;
-  setIsApiModalOpen: React.Dispatch<React.SetStateAction<boolean>>;
-}) {
-  return (
-    <div
-      className={`${
-        isMenuOpen ? 'w-72 lg:w-96' : 'w-16'
-      } fixed flex h-screen flex-col border-r-2 border-gray-100 bg-gray-50 transition-all`}
-    >
-      <div
-        className={`${
-          isMenuOpen ? 'w-full' : 'w-16'
-        } ml-auto h-16 border-b-2 border-gray-100`}
-      >
-        <button
-          className="float-right mr-5 mt-5 h-5 w-5"
-          onClick={() => setIsMenuOpen(!isMenuOpen)}
-        >
-          <img
-            src={Arrow1}
-            alt="menu toggle"
-            className={`${
-              isMenuOpen ? 'rotate-0' : 'rotate-180'
-            } m-auto w-3 transition-all`}
-          />
-        </button>
-      </div>
-
-      {isMenuOpen && (
-        <>
-          <div className="flex-grow border-b-2 border-gray-100"></div>
-
-          <div className="flex h-16 flex-col border-b-2 border-gray-100">
-            <div
-              className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
-              onClick={() => setIsApiModalOpen(true)}
-            >
-              <img src={Key} alt="key" className="ml-2 w-6" />
-              <p className="my-auto text-eerie-black">Reset Key</p>
-            </div>
-          </div>
-
-          <div className="flex h-48 flex-col border-b-2 border-gray-100">
-            <div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
-              <img src={Info} alt="info" className="ml-2 w-5" />
-              <p className="my-auto text-eerie-black">About</p>
-            </div>
-
-            <div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
-              <img src={Link} alt="link" className="ml-2 w-5" />
-              <p className="my-auto text-eerie-black">Discord</p>
-            </div>
-
-            <div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
-              <img src={Link} alt="link" className="ml-2 w-5" />
-              <p className="my-auto text-eerie-black">Github</p>
-            </div>
-          </div>
-        </>
-      )}
-    </div>
-  );
-}
-
-export default function Navigation({
-  isMobile,
-  isMenuOpen,
-  setIsMenuOpen,
-  setIsApiModalOpen,
-}: {
-  isMobile: boolean;
-  isMenuOpen: boolean;
-  setIsMenuOpen: React.Dispatch<React.SetStateAction<boolean>>;
-  setIsApiModalOpen: React.Dispatch<React.SetStateAction<boolean>>;
-}) {
-  if (isMobile) {
-    return <MobileNavigation />;
-  } else {
-    return (
-      <DesktopNavigation
-        isMenuOpen={isMenuOpen}
-        setIsMenuOpen={setIsMenuOpen}
-        setIsApiModalOpen={setIsApiModalOpen}
-      />
-    );
-  }
-}

frontend/src/components/Navigation/PastChat.tsx

@@ -1 +0,0 @@
-export default function PastChat() {}

frontend/src/conversation/Conversation.tsx

@@ -0,0 +1,73 @@
+import { useEffect, useRef } from 'react';
+import { useDispatch, useSelector } from 'react-redux';
+import Hero from '../Hero';
+import { AppDispatch } from '../store';
+import ConversationBubble from './ConversationBubble';
+import {
+  addMessage,
+  fetchAnswer,
+  selectConversation,
+  selectStatus,
+} from './conversationSlice';
+import Send from './../assets/send.svg';
+import Spinner from './../assets/spinner.svg';
+
+export default function Conversation() {
+  const messages = useSelector(selectConversation);
+  const status = useSelector(selectStatus);
+  const dispatch = useDispatch<AppDispatch>();
+  const endMessageRef = useRef<HTMLDivElement>(null);
+  const inputRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() =>
+    endMessageRef?.current?.scrollIntoView({ behavior: 'smooth' }),
+  );
+
+  const handleQuestion = (question: string) => {
+    dispatch(addMessage({ text: question, type: 'QUESTION' }));
+    dispatch(fetchAnswer({ question }));
+  };
+
+  return (
+    <div className="flex justify-center p-6">
+      <div className="w-10/12 transition-all md:w-1/2">
+        {messages.map((message, index) => {
+          return (
+            <ConversationBubble
+              ref={index === messages.length - 1 ? endMessageRef : null}
+              className="mb-7"
+              key={index}
+              message={message.text}
+              type={message.type}
+            ></ConversationBubble>
+          );
+        })}
+        {messages.length === 0 && <Hero className="mt-24"></Hero>}
+      </div>
+      <div className="fixed bottom-2 flex w-10/12 md:w-[50%]">
+        <div
+          ref={inputRef}
+          contentEditable
+          className={`min-h-5 border-000000 overflow-x-hidden; max-h-24 w-full overflow-y-auto rounded-xl border bg-white p-2 pr-9 opacity-100 focus:border-2 focus:outline-none`}
+        ></div>
+        {status === 'loading' ? (
+          <img
+            src={Spinner}
+            className="relative right-9 animate-spin cursor-pointer"
+          ></img>
+        ) : (
+          <img
+            onClick={() => {
+              if (inputRef.current?.textContent) {
+                handleQuestion(inputRef.current.textContent);
+                inputRef.current.textContent = '';
+              }
+            }}
+            src={Send}
+            className="relative right-9 cursor-pointer"
+          ></img>
+        )}
+      </div>
+    </div>
+  );
+}

frontend/src/conversation/ConversationBubble.tsx

@@ -0,0 +1,26 @@
+import { forwardRef } from 'react';
+import Avatar from '../Avatar';
+import { MESSAGE_TYPE } from './conversationModels';
+
+const ConversationBubble = forwardRef<
+  HTMLDivElement,
+  {
+    message: string;
+    type: MESSAGE_TYPE;
+    className: string;
+  }
+>(function ConversationBubble({ message, type, className }, ref) {
+  return (
+    <div
+      ref={ref}
+      className={`flex rounded-3xl ${
+        type === 'QUESTION' ? '' : 'bg-gray-1000'
+      } py-7 px-5 ${className}`}
+    >
+      <Avatar avatar={type === 'QUESTION' ? '👤' : '🦖'}></Avatar>
+      <p className="ml-5">{message}</p>
+    </div>
+  );
+});
+
+export default ConversationBubble;

frontend/src/conversation/conversationApi.ts

@@ -0,0 +1,24 @@
+import { Answer } from './conversationModels';
+
+export function fetchAnswerApi(
+  question: string,
+  apiKey: string,
+): Promise<Answer> {
+  // a mock answer generator, this is going to be replaced with real http call
+  return new Promise((resolve) => {
+    setTimeout(() => {
+      let result = '';
+      const characters =
+        'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+      const charactersLength = characters.length;
+      let counter = 0;
+      while (counter < 5) {
+        result += characters.charAt(
+          Math.floor(Math.random() * charactersLength),
+        );
+        counter += 1;
+      }
+      resolve({ answer: result, query: question, result });
+    }, 3000);
+  });
+}

frontend/src/conversation/conversationModels.ts

@@ -0,0 +1,18 @@
+export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER';
+export type Status = 'idle' | 'loading' | 'failed';
+
+export interface Message {
+  text: string;
+  type: MESSAGE_TYPE;
+}
+
+export interface ConversationState {
+  conversation: Message[];
+  status: Status;
+}
+
+export interface Answer {
+  answer: string;
+  query: string;
+  result: string;
+}

frontend/src/conversation/conversationSlice.ts

@@ -0,0 +1,55 @@
+import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';
+import store from '../store';
+import { fetchAnswerApi } from './conversationApi';
+import { Answer, ConversationState, Message } from './conversationModels';
+
+const initialState: ConversationState = {
+  conversation: [],
+  status: 'idle',
+};
+
+export const fetchAnswer = createAsyncThunk<
+  Answer,
+  { question: string },
+  { state: RootState }
+>('fetchAnswer', async ({ question }, { getState }) => {
+  const state = getState();
+  const answer = await fetchAnswerApi(question, state.preference.apiKey);
+  return answer;
+});
+
+export const conversationSlice = createSlice({
+  name: 'conversation',
+  initialState,
+  reducers: {
+    addMessage(state, action: PayloadAction<Message>) {
+      state.conversation.push(action.payload);
+    },
+  },
+  extraReducers(builder) {
+    builder
+      .addCase(fetchAnswer.pending, (state) => {
+        state.status = 'loading';
+      })
+      .addCase(fetchAnswer.fulfilled, (state, action) => {
+        state.status = 'idle';
+        state.conversation.push({
+          text: action.payload.answer,
+          type: 'ANSWER',
+        });
+      })
+      .addCase(fetchAnswer.rejected, (state) => {
+        state.status = 'failed';
+      });
+  },
+});
+
+type RootState = ReturnType<typeof store.getState>;
+
+export const selectConversation = (state: RootState) =>
+  state.conversation.conversation;
+
+export const selectStatus = (state: RootState) => state.conversation.status;
+
+export const { addMessage } = conversationSlice.actions;
+export default conversationSlice.reducer;

frontend/src/index.css

@@ -15,6 +15,8 @@
 html {
   line-height: 1.15; /* 1 */
   -webkit-text-size-adjust: 100%; /* 2 */
+  min-height: 100vh;
+  overflow-x: hidden;
 }
 
 /* Sections
@@ -26,6 +28,9 @@ html {
 
 body {
   margin: 0;
+  min-height: 100vh;
+  overflow-x: hidden;
+  font-family: 'Inter', sans-serif;
 }
 
 /**

frontend/src/main.tsx

@@ -2,12 +2,16 @@ import React from 'react';
 import ReactDOM from 'react-dom/client';
 import App from './App';
 import { BrowserRouter } from 'react-router-dom';
+import { Provider } from 'react-redux';
+import store from './store';
 import './index.css';
 
 ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render(
   <React.StrictMode>
     <BrowserRouter>
-      <App />
+      <Provider store={store}>
+        <App />
+      </Provider>
     </BrowserRouter>
   </React.StrictMode>,
 );

frontend/src/models/misc.ts

@@ -0,0 +1,5 @@
+export type ActiveState = 'ACTIVE' | 'INACTIVE';
+
+export type User = {
+  avatar: string;
+};

frontend/src/preferences/APIKeyModal.tsx

@@ -0,0 +1,83 @@
+import { useState } from 'react';
+import { useDispatch } from 'react-redux';
+import { ActiveState } from '../models/misc';
+import { setApiKey } from './preferenceSlice';
+
+export default function APIKeyModal({
+  modalState,
+  setModalState,
+  isCancellable = true,
+}: {
+  modalState: ActiveState;
+  setModalState: (val: ActiveState) => void;
+  isCancellable?: boolean;
+}) {
+  const dispatch = useDispatch();
+  const [key, setKey] = useState('');
+  const [isError, setIsError] = useState(false);
+
+  function handleSubmit() {
+    if (key.length <= 1) {
+      setIsError(true);
+    } else {
+      dispatch(setApiKey(key));
+      setModalState('INACTIVE');
+      setKey('');
+      setIsError(false);
+    }
+  }
+
+  function handleCancel() {
+    setKey('');
+    setIsError(false);
+    setModalState('INACTIVE');
+  }
+
+  return (
+    <div
+      className={`${
+        modalState === 'ACTIVE' ? 'visible' : 'hidden'
+      } absolute z-30  h-screen w-screen  bg-gray-alpha`}
+    >
+      <article className="mx-auto mt-24 flex w-[90vw] max-w-lg  flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
+        <p className="text-xl text-jet">OpenAI API Key</p>
+        <p className="text-lg leading-5 text-gray-500">
+          Before you can start using DocsGPT we need you to provide an API key
+          for llm. Currently, we support only OpenAI but soon many more. You can
+          find it here.
+        </p>
+        <input
+          type="text"
+          className="h-10 w-full border-b-2 border-jet focus:outline-none"
+          value={key}
+          maxLength={100}
+          placeholder="API Key"
+          onChange={(e) => setKey(e.target.value)}
+        />
+        <div className="flex flex-row-reverse">
+          <div>
+            <button
+              onClick={() => handleSubmit()}
+              className="ml-auto h-10 w-20 rounded-lg bg-violet-800 text-white transition-all hover:bg-violet-700"
+            >
+              Save
+            </button>
+            {isCancellable && (
+              <button
+                onClick={() => handleCancel()}
+                className="ml-5 h-10 w-20 rounded-lg border border-violet-700 bg-white text-violet-800 transition-all hover:bg-violet-700 hover:text-white"
+              >
+                Cancel
+              </button>
+            )}
+          </div>
+          {isError && (
+            <p className="mr-auto text-sm text-red-500">
+              Please enter a valid API key
+            </p>
+          )}
+        </div>
+      </article>
+    </div>
+  );
+}

frontend/src/preferences/SelectDocsModal.tsx

@@ -0,0 +1,132 @@
+import { useEffect, useState } from 'react';
+import { useDispatch, useSelector } from 'react-redux';
+import { ActiveState } from '../models/misc';
+import {
+  setSelectedDocs,
+  setSourceDocs,
+  selectSourceDocs,
+} from './preferenceSlice';
+import { getDocs, Doc } from './selectDocsApi';
+
+export default function APIKeyModal({
+  modalState,
+  setModalState,
+  isCancellable = true,
+}: {
+  modalState: ActiveState;
+  setModalState: (val: ActiveState) => void;
+  isCancellable?: boolean;
+}) {
+  const dispatch = useDispatch();
+  const docs = useSelector(selectSourceDocs);
+  const [localSelectedDocs, setLocalSelectedDocs] = useState<Doc | null>(null);
+  const [isDocsListOpen, setIsDocsListOpen] = useState(false);
+  const [isError, setIsError] = useState(false);
+
+  function handleSubmit() {
+    if (!localSelectedDocs) {
+      setIsError(true);
+    } else {
+      dispatch(setSelectedDocs(localSelectedDocs));
+      setModalState('INACTIVE');
+      setLocalSelectedDocs(null);
+      setIsError(false);
+    }
+  }
+
+  function handleCancel() {
+    setLocalSelectedDocs(null);
+    setIsError(false);
+    setModalState('INACTIVE');
+  }
+
+  useEffect(() => {
+    async function requestDocs() {
+      const data = await getDocs();
+      dispatch(setSourceDocs(data));
+    }
+
+    requestDocs();
+  }, []);
+
+  return (
+    <div
+      className={`${
+        modalState === 'ACTIVE' ? 'visible' : 'hidden'
+      } absolute z-30  h-screen w-screen  bg-gray-alpha`}
+    >
+      <article className="mx-auto mt-24 flex w-[90vw] max-w-lg  flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
+        <p className="text-xl text-jet">Select Source Documentation</p>
+        <p className="text-lg leading-5 text-gray-500">
+          Please select the library of documentation that you would like to use
+          with our app.
+        </p>
+        <div className="relative">
+          <div
+            className="h-10 w-full cursor-pointer border-b-2"
+            onClick={() => setIsDocsListOpen(!isDocsListOpen)}
+          >
+            {!localSelectedDocs ? (
+              <p className="py-3 text-gray-500">Select</p>
+            ) : (
+              <p className="py-3">
+                {localSelectedDocs.name} {localSelectedDocs.version}
+              </p>
+            )}
+          </div>
+          {isDocsListOpen && (
+            <div className="absolute top-10 left-0 max-h-52 w-full overflow-y-scroll bg-white">
+              {docs ? (
+                docs.map((doc, index) => {
+                  if (doc.model) {
+                    return (
+                      <div
+                        key={index}
+                        onClick={() => {
+                          setLocalSelectedDocs(doc);
+                          setIsDocsListOpen(false);
+                        }}
+                        className="h-10 w-full cursor-pointer border-x-2 border-b-2 hover:bg-gray-100"
+                      >
+                        <p className="ml-5 py-3">
+                          {doc.name} {doc.version}
+                        </p>
+                      </div>
+                    );
+                  }
+                })
+              ) : (
+                <div className="h-10 w-full cursor-pointer border-x-2 border-b-2 hover:bg-gray-100">
+                  <p className="ml-5 py-3">No default documentation.</p>
+                </div>
+              )}
+            </div>
+          )}
+        </div>
+        <div className="flex flex-row-reverse">
+          {isCancellable && (
+            <button
+              onClick={() => handleCancel()}
+              className="ml-5 h-10 w-20 rounded-lg border border-violet-700 bg-white text-violet-800 transition-all hover:bg-violet-700 hover:text-white"
+            >
+              Cancel
+            </button>
+          )}
+          <button
+            onClick={() => {
+              handleSubmit();
+            }}
+            className="ml-auto h-10 w-20 rounded-lg bg-violet-800 text-white transition-all hover:bg-violet-700"
+          >
+            Save
+          </button>
+          {isError && (
+            <p className="mr-auto text-sm text-red-500">
+              Please select source documentation.
+            </p>
+          )}
+        </div>
+      </article>
+    </div>
+  );
+}

frontend/src/preferences/preferenceSlice.ts

@@ -0,0 +1,44 @@
+import { createSlice } from '@reduxjs/toolkit';
+import { Doc } from './selectDocsApi';
+import store from '../store';
+
+interface Preference {
+  apiKey: string;
+  selectedDocs: Doc | null;
+  sourceDocs: Doc[] | null;
+}
+
+const initialState: Preference = {
+  apiKey: '',
+  selectedDocs: null,
+  sourceDocs: null,
+};
+
+export const prefSlice = createSlice({
+  name: 'preference',
+  initialState,
+  reducers: {
+    setApiKey: (state, action) => {
+      state.apiKey = action.payload;
+    },
+    setSelectedDocs: (state, action) => {
+      state.selectedDocs = action.payload;
+    },
+    setSourceDocs: (state, action) => {
+      state.sourceDocs = action.payload;
+    },
+  },
+});
+
+export const { setApiKey, setSelectedDocs, setSourceDocs } = prefSlice.actions;
+export default prefSlice.reducer;
+
+type RootState = ReturnType<typeof store.getState>;
+
+export const selectApiKey = (state: RootState) => state.preference.apiKey;
+export const selectApiKeyStatus = (state: RootState) =>
+  !!state.preference.apiKey;
+export const selectSelectedDocsStatus = (state: RootState) =>
+  !!state.preference.selectedDocs;
+export const selectSourceDocs = (state: RootState) =>
+  state.preference.sourceDocs;

frontend/src/preferences/selectDocsApi.ts

@@ -0,0 +1,33 @@
+//Exporting Doc type from here since its the first place its used and seems needless to make an entire file for it.
+export type Doc = {
+  name: string;
+  language: string;
+  version: string;
+  description: string;
+  fullName: string;
+  dat: string;
+  docLink: string;
+  model: string;
+};
+
+//Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
+export async function getDocs(): Promise<Doc[] | null> {
+  try {
+    //Fetch default source docs
+    const response = await fetch(
+      'https://d3dg1063dc54p9.cloudfront.net/combined.json',
+    );
+    const data = await response.json();
+
+    //Create array of Doc objects
+    const docs: Doc[] = [];
+
+    data.forEach((doc: object) => {
+      docs.push(doc as Doc);
+    });
+
+    return docs;
+  } catch (error) {
+    return null;
+  }
+}

frontend/src/store.ts

@@ -0,0 +1,13 @@
+import { configureStore } from '@reduxjs/toolkit';
+import { conversationSlice } from './conversation/conversationSlice';
+import { prefSlice } from './preferences/preferenceSlice';
+
+const store = configureStore({
+  reducer: {
+    preference: prefSlice.reducer,
+    conversation: conversationSlice.reducer,
+  },
+});
+
+export type AppDispatch = typeof store.dispatch;
+export default store;

frontend/tailwind.config.cjs

@@ -11,6 +11,7 @@ module.exports = {
         'eerie-black': '#212121',
         jet: '#343541',
         'gray-alpha': 'rgba(0,0,0, .1)',
+        'gray-1000': '#F6F6F6',
       },
     },
   },

scripts/ingest.py

@@ -1,6 +1,11 @@
+from collections import defaultdict
+import os
 import sys
 import nltk
 import dotenv
+import typer
+
+from typing import List, Optional
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 
@@ -10,28 +15,68 @@ from parser.open_ai_func import call_openai_api, get_user_permission
 
 dotenv.load_dotenv()
 
-#Specify your folder HERE
-directory_to_ingest = 'inputs'
+app = typer.Typer(add_completion=False)
+
+nltk.download('punkt', quiet=True)
+nltk.download('averaged_perceptron_tagger', quiet=True)
 
-nltk.download('punkt')
-nltk.download('averaged_perceptron_tagger')
 
 #Splits all files in specified folder to documents
-raw_docs = SimpleDirectoryReader(input_dir=directory_to_ingest).load_data()
-raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
-# Here we split the documents, as needed, into smaller chunks.
-# We do this due to the context limits of the LLMs.
-text_splitter = RecursiveCharacterTextSplitter()
-docs = text_splitter.split_documents(raw_docs)
+@app.command()
+def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
+                                                   help="Whether to skip price confirmation"),
+           dir: Optional[List[str]] = typer.Option(["inputs"],
+                                                   help="""List of paths to directory for index creation.
+                                                        E.g. --dir inputs --dir inputs2"""),
+           file: Optional[List[str]] = typer.Option(None,
+                                                   help="""File paths to use (Optional; overrides dir).
+                                                        E.g. --file inputs/1.md --file inputs/2.md"""),
+           recursive: Optional[bool] = typer.Option(True,
+                                                   help="Whether to recursively search in subdirectories."),
+           limit: Optional[int] = typer.Option(None,
+                                                   help="Maximum number of files to read."),
+           formats: Optional[List[str]] = typer.Option([".rst", ".md"],
+                                                   help="""List of required extensions (list with .)
+                                                        Currently supported: .rst, .md, .pdf, .docx, .csv, .epub"""),
+           exclude: Optional[bool] = typer.Option(True, help="Whether to exclude hidden files (dotfiles).")):
 
-# Here we check for command line arguments for bot calls.
-# If no argument exists or the permission_bypass_flag argument is not '-y',
-# user permission is requested to call the API.
-if len(sys.argv) > 1:
-    permission_bypass_flag = sys.argv[1]
-    if permission_bypass_flag == '-y':
-        call_openai_api(docs)
-    else:
-        get_user_permission(docs)
-else:
-    get_user_permission(docs)
+    """
+        Creates index from specified location or files.
+        By default /inputs folder is used, .rst and .md are parsed.
+    """
+
+    def process_one_docs(directory, folder_name):
+        raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=file, recursive=recursive,
+                                         required_exts=formats, num_files_limit=limit,
+                                         exclude_hidden=exclude).load_data()
+        raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+        # Here we split the documents, as needed, into smaller chunks.
+        # We do this due to the context limits of the LLMs.
+        text_splitter = RecursiveCharacterTextSplitter()
+        docs = text_splitter.split_documents(raw_docs)
+
+        # Here we check for command line arguments for bot calls.
+        # If no argument exists or the yes is not True, then the
+        # user permission is requested to call the API.
+        if len(sys.argv) > 1:
+            if yes:
+                call_openai_api(docs, folder_name)
+            else:
+                get_user_permission(docs, folder_name)
+        else:
+            get_user_permission(docs, folder_name)
+
+    folder_counts = defaultdict(int)
+    folder_names = []
+    for dir_path in dir:
+        folder_name = os.path.basename(os.path.normpath(dir_path))
+        folder_counts[folder_name] += 1
+        if folder_counts[folder_name] > 1:
+            folder_name = f"{folder_name}_{folder_counts[folder_name]}"
+        folder_names.append(folder_name)
+
+    for directory, folder_name in zip(dir, folder_names):
+        process_one_docs(directory, folder_name)
+
+if __name__ == "__main__":
+  app()

scripts/old/ingest_rst_sphinx.py

@@ -29,6 +29,18 @@ def convert_rst_to_txt(src_dir, dst_dir):
                f"-D source_suffix=.rst " \
                f"-C {dst_dir} "
         sphinx_main(args.split())
+      elif file.endswith(".md"):
+        # Rename the .md file to .rst file
+        src_file = os.path.join(root, file)
+        dst_file = os.path.join(root, file.replace(".md", ".rst"))
+        os.rename(src_file, dst_file)
+        # Convert the .rst file to .txt file using sphinx-build
+        args = f". -b text -D extensions=sphinx.ext.autodoc " \
+                f"-D master_doc={dst_file} " \
+                f"-D source_suffix=.rst " \
+                f"-C {dst_dir} "
+        sphinx_main(args.split())
+
 
 def num_tokens_from_string(string: str, encoding_name: str) -> int:
 # Function to convert string to tokens and estimate user cost.

scripts/parser/file/rst_parser.py

@@ -24,6 +24,8 @@ class RstParser(BaseParser):
         remove_hyperlinks: bool = True,
         remove_images: bool = True,
         remove_table_excess: bool = True,
+        remove_interpreters: bool = True,
+        remove_directives: bool = True,
         remove_whitespaces_excess: bool = True,
         #Be carefull with remove_characters_excess, might cause data loss
         remove_characters_excess: bool = True,
@@ -34,6 +36,8 @@ class RstParser(BaseParser):
         self._remove_hyperlinks = remove_hyperlinks
         self._remove_images = remove_images
         self._remove_table_excess = remove_table_excess
+        self._remove_interpreters = remove_interpreters
+        self._remove_directives = remove_directives
         self._remove_whitespaces_excess = remove_whitespaces_excess
         self._remove_characters_excess = remove_characters_excess
 
@@ -95,6 +99,18 @@ class RstParser(BaseParser):
         content = re.sub(pattern, r"\1", content)
         return content
 
+    def remove_directives(self, content: str) -> str:
+        """Removes reStructuredText Directives"""
+        pattern = r"`\.\.([^:]+)::"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_interpreters(self, content: str) -> str:
+        """Removes reStructuredText Interpreted Text Roles"""
+        pattern = r":(\w+):"
+        content = re.sub(pattern, "", content)
+        return content
+
     def remove_table_excess(self, content: str) -> str:
         """Pattern to remove grid table separators"""
         pattern = r"^\+[-]+\+[-]+\+$"
@@ -129,6 +145,10 @@ class RstParser(BaseParser):
             content = self.remove_images(content)
         if self._remove_table_excess:
             content = self.remove_table_excess(content)
+        if self._remove_directives:
+            content = self.remove_directives(content)
+        if self._remove_interpreters:
+            content = self.remove_interpreters(content)
         rst_tups = self.rst_to_tups(content)
         if self._remove_whitespaces_excess:
             rst_tups = self.remove_whitespaces_excess(rst_tups)

scripts/parser/open_ai_func.py

@@ -1,9 +1,17 @@
+import os
 import faiss
 import pickle
 import tiktoken
 from langchain.vectorstores import FAISS
 from langchain.embeddings import OpenAIEmbeddings
 
+#from langchain.embeddings import HuggingFaceEmbeddings
+#from langchain.embeddings import HuggingFaceInstructEmbeddings
+#from langchain.embeddings import CohereEmbeddings
+
+from retry import retry
+
+
 
 def num_tokens_from_string(string: str, encoding_name: str) -> int:
 # Function to convert string to tokens and estimate user cost.
@@ -12,16 +20,44 @@ def num_tokens_from_string(string: str, encoding_name: str) -> int:
     total_price = ((num_tokens/1000) * 0.0004)
     return num_tokens, total_price
 
-def call_openai_api(docs):
+@retry(tries=10, delay=60)
+def store_add_texts_with_retry(store, i):
+    store.add_texts([i.page_content], metadatas=[i.metadata])
+
+def call_openai_api(docs, folder_name):
 # Function to create a vector store from the documents and save it to disk.
-    store = FAISS.from_documents(docs, OpenAIEmbeddings())
-    faiss.write_index(store.index, "docs.index")
-    store.index = None
 
-    with open("faiss_store.pkl", "wb") as f:
-        pickle.dump(store, f)
+    # create output folder if it doesn't exist
+    if not os.path.exists(f"outputs/{folder_name}"):
+        os.makedirs(f"outputs/{folder_name}")
 
-def get_user_permission(docs):
+    from tqdm import tqdm
+    docs_test = [docs[0]]
+    # remove the first element from docs
+    docs.pop(0)
+    # cut first n docs if you want to restart
+    #docs = docs[:n]
+    c1 = 0
+    store = FAISS.from_documents(docs_test, OpenAIEmbeddings())
+
+    # Uncomment for MPNet embeddings
+    # model_name = "sentence-transformers/all-mpnet-base-v2"
+    # hf = HuggingFaceEmbeddings(model_name=model_name)
+    # store = FAISS.from_documents(docs_test, hf)
+    for i in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs), bar_format='{l_bar}{bar}| Time Left: {remaining}'):
+        try:
+            store_add_texts_with_retry(store, i)
+        except Exception as e:
+            print(e)
+            print("Error on ", i)
+            print("Saving progress")
+            print(f"stopped at {c1} out of {len(docs)}")
+            store.save_local(f"outputs/{folder_name}")
+            break
+        c1 += 1
+    store.save_local(f"outputs/{folder_name}")
+
+def get_user_permission(docs, folder_name):
 # Function to ask user permission to call the OpenAI api and spend their OpenAI funds.
     # Here we convert the docs list to a string and calculate the number of OpenAI tokens the string represents.
     #docs_content = (" ".join(docs))
@@ -37,8 +73,8 @@ def get_user_permission(docs):
     #Here we check for user permission before calling the API.
     user_input = input("Price Okay? (Y/N) \n").lower()
     if user_input == "y":
-        call_openai_api(docs)
+        call_openai_api(docs, folder_name)
     elif user_input == "":
-        call_openai_api(docs)
+        call_openai_api(docs, folder_name)
     else:
-        print("The API was not called. No money was spent.")
+        print("The API was not called. No money was spent.")