Merge branch 'arc53:main' into main

README.md

@@ -61,5 +61,7 @@ Copy .env_sample and create .env with your openai api token
 
 ## [How to use any other documentation](https://github.com/arc53/docsgpt/wiki/How-to-train-on-other-documentation)
 
+## [How to host it locally (so all data will stay on-premises)](https://github.com/arc53/DocsGPT/wiki/How-to-use-different-LLM's#hosting-everything-locally)
+
 Built with [🦜️🔗 LangChain](https://github.com/hwchase17/langchain)
 

application/app.py

@@ -1,54 +1,88 @@
 import os
-import pickle
+import json
+
 import dotenv
-import datetime
-from flask import Flask, request, render_template
-# os.environ["LANGCHAIN_HANDLER"] = "langchain"
-import faiss
-from langchain import OpenAI, VectorDBQA
-from langchain.chains.question_answering import load_qa_chain
-from langchain.prompts import PromptTemplate
 import requests
+from flask import Flask, request, render_template
+from langchain import FAISS
+from langchain import OpenAI, VectorDBQA, HuggingFaceHub, Cohere
+from langchain.chains.question_answering import load_qa_chain
+from langchain.embeddings import OpenAIEmbeddings, HuggingFaceHubEmbeddings, CohereEmbeddings, HuggingFaceInstructEmbeddings
+from langchain.prompts import PromptTemplate
+
+# os.environ["LANGCHAIN_HANDLER"] = "langchain"
+
+if os.getenv("LLM_NAME") is not None:
+    llm_choice = os.getenv("LLM_NAME")
+else:
+    llm_choice = "openai"
+
+if os.getenv("EMBEDDINGS_NAME") is not None:
+    embeddings_choice = os.getenv("EMBEDDINGS_NAME")
+else:
+    embeddings_choice = "openai_text-embedding-ada-002"
+
+
+
+if llm_choice == "manifest":
+    from manifest import Manifest
+    from langchain.llms.manifest import ManifestWrapper
+
+    manifest = Manifest(
+        client_name="huggingface",
+        client_connection="http://127.0.0.1:5000"
+    )
 
 # Redirect PosixPath to WindowsPath on Windows
 import platform
+
 if platform.system() == "Windows":
     import pathlib
+
     temp = pathlib.PosixPath
     pathlib.PosixPath = pathlib.WindowsPath
 
 # loading the .env file
 dotenv.load_dotenv()
 
-
 with open("combine_prompt.txt", "r") as f:
     template = f.read()
 
-# check if OPENAI_API_KEY is set
-if os.getenv("OPENAI_API_KEY") is not None:
-    api_key_set = True
+with open("combine_prompt_hist.txt", "r") as f:
+    template_hist = f.read()
 
+if os.getenv("API_KEY") is not None:
+    api_key_set = True
 else:
     api_key_set = False
-
-
+if os.getenv("EMBEDDINGS_KEY") is not None:
+    embeddings_key_set = True
+else:
+    embeddings_key_set = False
 
 app = Flask(__name__)
 
 
 @app.route("/")
 def home():
-    return render_template("index.html", api_key_set=api_key_set)
+    return render_template("index.html", api_key_set=api_key_set, llm_choice=llm_choice,
+                           embeddings_choice=embeddings_choice)
 
 
 @app.route("/api/answer", methods=["POST"])
 def api_answer():
     data = request.get_json()
     question = data["question"]
+    history = data["history"]
     if not api_key_set:
         api_key = data["api_key"]
     else:
-        api_key = os.getenv("OPENAI_API_KEY")
+        api_key = os.getenv("API_KEY")
+    if not embeddings_key_set:
+        embeddings_key = data["embeddings_key"]
+    else:
+        embeddings_key = os.getenv("EMBEDDINGS_KEY")
+
 
     # check if the vectorstore is set
     if "active_docs" in data:
@@ -59,25 +93,37 @@ def api_answer():
         vectorstore = ""
 
     # loading the index and the store and the prompt template
-    index = faiss.read_index(f"{vectorstore}docs.index")
+    # Note if you have used other embeddings than OpenAI, you need to change the embeddings
+    if embeddings_choice == "openai_text-embedding-ada-002":
+        docsearch = FAISS.load_local(vectorstore, OpenAIEmbeddings(openai_api_key=embeddings_key))
+    elif embeddings_choice == "huggingface_sentence-transformers/all-mpnet-base-v2":
+        docsearch = FAISS.load_local(vectorstore, HuggingFaceHubEmbeddings())
+    elif embeddings_choice == "huggingface_hkunlp/instructor-large":
+        docsearch = FAISS.load_local(vectorstore, HuggingFaceInstructEmbeddings())
+    elif embeddings_choice == "cohere_medium":
+        docsearch = FAISS.load_local(vectorstore, CohereEmbeddings(cohere_api_key=embeddings_key))
 
-    with open(f"{vectorstore}faiss_store.pkl", "rb") as f:
-        store = pickle.load(f)
-
-    store.index = index
     # create a prompt template
-    c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template)
-    # create a chain with the prompt template and the store
+    if history:
+        history = json.loads(history)
+        template_temp = template_hist.replace("{historyquestion}", history[0]).replace("{historyanswer}", history[1])
+        c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template_temp)
+    else:
+        c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template)
 
-    #chain = VectorDBQA.from_llm(llm=OpenAI(openai_api_key=api_key, temperature=0), vectorstore=store, combine_prompt=c_prompt)
-    # chain = VectorDBQA.from_chain_type(llm=OpenAI(openai_api_key=api_key, temperature=0), chain_type='map_reduce',
-    #                                    vectorstore=store)
+    if llm_choice == "openai":
+        llm = OpenAI(openai_api_key=api_key, temperature=0)
+    elif llm_choice == "manifest":
+        llm = ManifestWrapper(client=manifest, llm_kwargs={"temperature": 0.001, "max_tokens": 2048})
+    elif llm_choice == "huggingface":
+        llm = HuggingFaceHub(repo_id="bigscience/bloom", huggingfacehub_api_token=api_key)
+    elif llm_choice == "cohere":
+        llm = Cohere(model="command-xlarge-nightly", cohere_api_key=api_key)
 
-    qa_chain = load_qa_chain(OpenAI(openai_api_key=api_key, temperature=0), chain_type="map_reduce",
+    qa_chain = load_qa_chain(llm=llm, chain_type="map_reduce",
                              combine_prompt=c_prompt)
-    chain = VectorDBQA(combine_documents_chain=qa_chain, vectorstore=store)
-
 
+    chain = VectorDBQA(combine_documents_chain=qa_chain, vectorstore=docsearch, k=4)
 
     # fetch the answer
     result = chain({"query": question})
@@ -94,6 +140,7 @@ def api_answer():
     # }
     return result
 
+
 @app.route("/api/docs_check", methods=["POST"])
 def check_docs():
     # check if docs exist in a vectorstore folder
@@ -104,20 +151,21 @@ def check_docs():
     if os.path.exists(vectorstore):
         return {"status": 'exists'}
     else:
-        r = requests.get(base_path + vectorstore + "docs.index")
+        r = requests.get(base_path + vectorstore + "index.faiss")
         # save to vectors directory
         # check if the directory exists
         if not os.path.exists(vectorstore):
             os.makedirs(vectorstore)
 
-        with open(vectorstore + "docs.index", "wb") as f:
+        with open(vectorstore + "index.faiss", "wb") as f:
             f.write(r.content)
         # download the store
-        r = requests.get(base_path + vectorstore + "faiss_store.pkl")
-        with open(vectorstore + "faiss_store.pkl", "wb") as f:
+        r = requests.get(base_path + vectorstore + "index.pkl")
+        with open(vectorstore + "index.pkl", "wb") as f:
             f.write(r.content)
 
-        return {"status":  'loaded'}
+        return {"status": 'loaded'}
+
 
 # handling CORS
 @app.after_request

application/combine_prompt_hist.txt

@@ -0,0 +1,27 @@
+You are a DocsGPT bot assistant by Arc53 that provides help with programming libraries. You give thorough answers with code examples.
+Given the following extracted parts of a long document and a question, create a final answer with references ("SOURCES").
+ALWAYS return a "SOURCES" part in your answer. You can also remeber things from previous questions and use them in your answer.
+
+QUESTION: How to merge tables in pandas?
+=========
+Content: pandas provides various facilities for easily combining together Series or DataFrame with various kinds of set logic for the indexes and relational algebra functionality in the case of join / merge-type operations.
+Source: 28-pl
+Content: pandas provides a single function, merge(), as the entry point for all standard database join operations between DataFrame or named Series objects: \n\npandas.merge(left, right, how='inner', on=None, left_on=None, right_on=None, left_index=False, right_index=False, sort=False, suffixes=('_x', '_y'), copy=True, indicator=False, validate=None)
+Source: 30-pl
+=========
+FINAL ANSWER: To merge two tables in pandas, you can use the pd.merge() function. The basic syntax is: \n\npd.merge(left, right, on, how) \n\nwhere left and right are the two tables to merge, on is the column to merge on, and how is the type of merge to perform. \n\nFor example, to merge the two tables df1 and df2 on the column 'id', you can use: \n\npd.merge(df1, df2, on='id', how='inner')
+SOURCES: 28-pl 30-pl
+
+QUESTION: {historyquestion}
+=========
+CONTENT:
+SOURCE:
+=========
+FINAL ANSWER: {historyanswer}
+SOURCES:
+
+QUESTION: {question}
+=========
+{summaries}
+=========
+FINAL ANSWER:

application/requirements.txt

@@ -45,6 +45,7 @@ pytz==2022.7.1
 PyYAML==6.0
 regex==2022.10.31
 requests==2.28.2
+retry==0.9.2
 six==1.16.0
 snowballstemmer==2.2.0
 Sphinx==6.1.3
@@ -64,6 +65,6 @@ typer==0.7.0
 typing-inspect==0.8.0
 typing_extensions==4.4.0
 urllib3==1.26.14
-Werkzeug==2.2.2
+Werkzeug==2.2.3
 XlsxWriter==3.0.8
 yarl==1.8.2

application/static/src/chat.js

@@ -25,6 +25,8 @@ if (el) {
 
             body: JSON.stringify({question: message,
                 api_key: localStorage.getItem('apiKey'),
+                embeddings_key: localStorage.getItem('apiKey'),
+                history: localStorage.getItem('chatHistory'),
                 active_docs: localStorage.getItem('activeDocs')}),
         })
             .then(response => response.json())
@@ -38,9 +40,12 @@ if (el) {
                 chatWindow.scrollTop = chatWindow.scrollHeight;
                 document.getElementById("button-submit").innerHTML = 'Send';
                 document.getElementById("button-submit").disabled = false;
+                let chatHistory = [message, data.answer];
+                localStorage.setItem('chatHistory', JSON.stringify(chatHistory));
             })
             .catch((error) => {
                 console.error('Error:', error);
+                console.log(error);
                 document.getElementById("button-submit").innerHTML = 'Send';
                 document.getElementById("button-submit").disabled = false;
             });

application/templates/index.html

@@ -131,13 +131,17 @@ This will return a new DataFrame with all the columns from both tables, and only
                     var option = document.createElement("option");
                     if (docsIndex[key].name == docsIndex[key].language) {
                         option.text = docsIndex[key].name + " " + docsIndex[key].version;
-                        option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/";
-                        select.add(option);
+                        option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
+                        if (docsIndex[key].model == "{{ embeddings_choice }}") {
+                            select.add(option);
+                        }
                     }
                     else {
                         option.text = docsIndex[key].name + " " + docsIndex[key].version;
-                        option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/";
-                        select.add(option);
+                        option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
+                        if (docsIndex[key].model == "{{ embeddings_choice }}") {
+                            select.add(option);
+                        }
                     }
                 }
 

frontend/.eslintrc.cjs

@@ -19,6 +19,12 @@ module.exports = {
   plugins: ['react'],
   rules: {
     'react/react-in-jsx-scope': 'off',
+    'prettier/prettier': [
+      'error',
+      {
+        endOfLine: 'auto',
+      },
+    ],
   },
   settings: {
     'import/parsers': {
@@ -34,10 +40,4 @@ module.exports = {
       },
     },
   },
-  'prettier/prettier': [
-    'error',
-    {
-      endOfLine: 'auto',
-    },
-  ],
-}
+};

frontend/src/About.tsx

@@ -1,71 +1,54 @@
-import { useSelector } from 'react-redux';
-import { useMediaQuery } from '../hooks';
-import { selectIsMenuOpen } from '../store';
-
-//TODO - Add hyperlinks to text
-//TODO - Styling
-
-export default function About() {
-  const isMobile = useMediaQuery('(max-width: 768px)');
-  const isMenuOpen = useSelector(selectIsMenuOpen);
-
-  return (
-    //Parent div for all content shown through App.tsx routing needs to have this styling. Might change when state management is updated.
-    <div
-      className={`${
-        isMobile
-          ? isMenuOpen
-            ? 'mt-80'
-            : 'mt-16'
-          : isMenuOpen
-          ? 'md:ml-72 lg:ml-96'
-          : 'ml-16'
-      } h-full w-full p-6 transition-all`}
-    >
-      <article className="mx-auto my-auto flex w-full max-w-6xl flex-col gap-6 rounded-lg bg-gray-100 p-6 text-jet lg:p-10 xl:p-16">
-        <p className="text-3xl font-semibold">About DocsGPT 🦖</p>
-        <p className="mt-4 text-xl font-bold">
-          Find the information in your documentation through AI-powered
-          open-source chatbot. Powered by GPT-3, Faiss and LangChain.
-        </p>
-
-        <div>
-          <p className="text-lg">
-            If you want to add your own documentation, please follow the
-            instruction below:
-          </p>
-          <p className="mt-4 text-lg">
-            1. Navigate to{' '}
-            <span className="bg-gray-200 italic"> /application</span> folder
-          </p>
-          <p className="mt-4 text-lg">
-            2. Install dependencies from{' '}
-            <span className="bg-gray-200 italic">
-              pip install -r requirements.txt
-            </span>
-          </p>
-          <p className="mt-4 text-lg">
-            3. Prepare a <span className="bg-gray-200 italic">.env</span> file.
-            Copy <span className="bg-gray-200 italic">.env_sample</span> and
-            create <span className="bg-gray-200 italic">.env</span> with your
-            OpenAI API token
-          </p>
-          <p className="mt-4 text-lg">
-            4. Run the app with{' '}
-            <span className="bg-gray-200 italic">python app.py</span>
-          </p>
-        </div>
-
-        <p className="text-lg">
-          Currently It uses python pandas documentation, so it will respond to
-          information relevant to pandas. If you want to train it on different
-          documentation - please follow this guide.
-        </p>
-
-        <p className="mt-4 text-lg">
-          If you want to launch it on your own server - follow this guide.
-        </p>
-      </article>
-    </div>
-  );
-}
+//TODO - Add hyperlinks to text
+//TODO - Styling
+
+export default function About() {
+  return (
+    //Parent div for all content shown through App.tsx routing needs to have this styling. Might change when state management is updated.
+    <div className="grid min-h-screen">
+      <article className=" mx-auto my-auto flex w-full max-w-6xl flex-col place-items-center gap-6 rounded-lg bg-gray-100 p-6 text-jet lg:p-10 xl:p-16">
+        <p className="text-3xl font-semibold">About DocsGPT 🦖</p>
+        <p className="mt-4 text-xl font-bold">
+          Find the information in your documentation through AI-powered
+          open-source chatbot. Powered by GPT-3, Faiss and LangChain.
+        </p>
+
+        <div>
+          <p className="text-lg">
+            If you want to add your own documentation, please follow the
+            instruction below:
+          </p>
+          <p className="mt-4 text-lg">
+            1. Navigate to{' '}
+            <span className="bg-gray-200 italic"> /application</span> folder
+          </p>
+          <p className="mt-4 text-lg">
+            2. Install dependencies from{' '}
+            <span className="bg-gray-200 italic">
+              pip install -r requirements.txt
+            </span>
+          </p>
+          <p className="mt-4 text-lg">
+            3. Prepare a <span className="bg-gray-200 italic">.env</span> file.
+            Copy <span className="bg-gray-200 italic">.env_sample</span> and
+            create <span className="bg-gray-200 italic">.env</span> with your
+            OpenAI API token
+          </p>
+          <p className="mt-4 text-lg">
+            4. Run the app with{' '}
+            <span className="bg-gray-200 italic">python app.py</span>
+          </p>
+        </div>
+
+        <p className="text-lg">
+          Currently It uses python pandas documentation, so it will respond to
+          information relevant to pandas. If you want to train it on different
+          documentation - please follow this guide.
+        </p>
+
+        <p className="mt-4 text-lg">
+          If you want to launch it on your own server - follow this guide.
+        </p>
+      </article>
+    </div>
+  );
+}

frontend/src/App.tsx

@@ -1,18 +1,29 @@
 import { Routes, Route } from 'react-router-dom';
-import Navigation from './components/Navigation';
-import Conversation from './components/Conversation/Conversation';
-import APIKeyModal from './components/APIKeyModal';
-import About from './components/About';
+import Navigation from './Navigation';
+import Conversation from './conversation/Conversation';
+import About from './About';
+import { useState } from 'react';
+import { ActiveState } from './models/misc';
 
 export default function App() {
+  const [navState, setNavState] = useState<ActiveState>('ACTIVE');
+
   return (
-    <div className="relative flex flex-col transition-all md:flex-row">
-      <APIKeyModal />
-      <Navigation />
-      <Routes>
-        <Route path="/" element={<Conversation />} />
-        <Route path="/about" element={<About />} />
-      </Routes>
+    <div className="min-h-full min-w-full">
+      <Navigation
+        navState={navState}
+        setNavState={(val: ActiveState) => setNavState(val)}
+      />
+      <div
+        className={`transition-all duration-200 ${
+          navState === 'ACTIVE' ? 'ml-0 md:ml-72 lg:ml-96' : ' ml-0 md:ml-16'
+        }`}
+      >
+        <Routes>
+          <Route path="/" element={<Conversation />} />
+          <Route path="/about" element={<About />} />
+        </Routes>
+      </div>
     </div>
   );
 }

frontend/src/Avatar.tsx

@@ -0,0 +1,9 @@
+export default function Avatar({
+  avatar,
+  size,
+}: {
+  avatar: string;
+  size?: 'SMALL' | 'MEDIUM' | 'LARGE';
+}) {
+  return <div>{avatar}</div>;
+}

frontend/src/Hero.tsx

@@ -0,0 +1,21 @@
+export default function Hero({ className = '' }: { className?: string }) {
+  return (
+    <div className={`flex flex-col ${className}`}>
+      <p className="mb-10 text-center text-4xl font-semibold">
+        DocsGPT <span className="text-3xl">🦖</span>
+      </p>
+      <p className="mb-3 text-center">
+        Welcome to DocsGPT, your technical documentation assistant!
+      </p>
+      <p className="mb-3 text-center">
+        Enter a query related to the information in the documentation you
+        selected to receive and we will provide you with the most relevant
+        answers.
+      </p>
+      <p className="text-center">
+        Start by entering your query in the input field below and we will do the
+        rest!
+      </p>
+    </div>
+  );
+}

frontend/src/Navigation.tsx

@@ -0,0 +1,97 @@
+import { NavLink } from 'react-router-dom';
+import Arrow1 from './assets/arrow.svg';
+import Hamburger from './assets/hamburger.svg';
+import Key from './assets/key.svg';
+import Info from './assets/info.svg';
+import Link from './assets/link.svg';
+import { ActiveState } from './models/misc';
+import APIKeyModal from './preferences/APIKeyModal';
+import { useSelector } from 'react-redux';
+import { selectApiKeyStatus } from './preferences/preferenceSlice';
+import { useState } from 'react';
+
+//TODO - Need to replace Chat button to open secondary nav with scrollable past chats option and new chat at top
+//TODO - Need to add Discord and Github links
+
+export default function Navigation({
+  navState,
+  setNavState,
+}: {
+  navState: ActiveState;
+  setNavState: (val: ActiveState) => void;
+}) {
+  const isApiKeySet = useSelector(selectApiKeyStatus);
+  const [apiKeyModalState, setApiKeyModalState] = useState<ActiveState>(
+    isApiKeySet ? 'INACTIVE' : 'ACTIVE',
+  );
+  return (
+    <>
+      <div
+        className={`${
+          navState === 'INACTIVE' && '-ml-96 md:-ml-60 lg:-ml-80'
+        } fixed z-10 flex h-full w-72 flex-col border-r-2 border-gray-100 bg-gray-50 transition-all duration-200 lg:w-96`}
+      >
+        <div className={'h-16 w-full border-b-2 border-gray-100'}>
+          <button
+            className="float-right mr-5 mt-5 h-5 w-5"
+            onClick={() =>
+              setNavState(navState === 'ACTIVE' ? 'INACTIVE' : 'ACTIVE')
+            }
+          >
+            <img
+              src={Arrow1}
+              alt="menu toggle"
+              className={`${
+                navState === 'INACTIVE' ? 'rotate-180' : 'rotate-0'
+              }  m-auto w-3 transition-all duration-200`}
+            />
+          </button>
+        </div>
+        <div className="flex-grow border-b-2 border-gray-100"></div>
+
+        <div className="flex h-16 flex-col border-b-2 border-gray-100">
+          <div
+            className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
+            onClick={() => {
+              setApiKeyModalState('ACTIVE');
+            }}
+          >
+            <img src={Key} alt="key" className="ml-2 w-6" />
+            <p className="my-auto text-eerie-black">Reset Key</p>
+          </div>
+        </div>
+
+        <div className="flex h-48 flex-col border-b-2 border-gray-100">
+          <NavLink
+            to="/about"
+            className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
+          >
+            <img src={Info} alt="info" className="ml-2 w-5" />
+            <p className="my-auto text-eerie-black">About</p>
+          </NavLink>
+
+          <div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
+            <img src={Link} alt="link" className="ml-2 w-5" />
+            <p className="my-auto text-eerie-black">Discord</p>
+          </div>
+
+          <div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
+            <img src={Link} alt="link" className="ml-2 w-5" />
+            <p className="my-auto text-eerie-black">Github</p>
+          </div>
+        </div>
+      </div>
+      <button
+        className="fixed mt-5 ml-6 h-6 w-6 md:hidden"
+        onClick={() => setNavState('ACTIVE')}
+      >
+        <img src={Hamburger} alt="menu toggle" className="w-7" />
+      </button>
+      <APIKeyModal
+        modalState={apiKeyModalState}
+        setModalState={setApiKeyModalState}
+        isCancellable={isApiKeySet}
+      />
+    </>
+  );
+}

frontend/src/assets/send.svg

@@ -0,0 +1,3 @@
+<svg width="21" height="18" viewBox="0 0 21 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0.00999999 18L21 9L0.00999999 0L0 7L15 9L0 11L0.00999999 18Z" fill="black" fill-opacity="0.54"/>
+</svg>

frontend/src/components/APIKeyModal.tsx

@@ -1,63 +0,0 @@
-import { useState } from 'react';
-import { useDispatch, useSelector } from 'react-redux';
-import {
-  setApiKey,
-  toggleApiKeyModal,
-  selectIsApiKeyModalOpen,
-} from '../store';
-
-export default function APIKeyModal({}) {
-  //TODO - Add form validation?
-  //TODO - Connect to backend
-  //TODO - Add link to OpenAI API Key page
-
-  const dispatch = useDispatch();
-  const isApiModalOpen = useSelector(selectIsApiKeyModalOpen);
-  const [key, setKey] = useState('');
-  const [formError, setFormError] = useState(false);
-
-  function handleSubmit() {
-    if (key.length < 1) {
-      setFormError(true);
-      return;
-    }
-    dispatch(setApiKey(key));
-    dispatch(toggleApiKeyModal());
-  }
-
-  return (
-    <div
-      className={`${
-        isApiModalOpen ? 'visible' : 'hidden'
-      } absolute z-30  h-screen w-screen  bg-gray-alpha`}
-    >
-      <article className="mx-auto mt-24 flex w-[90vw] max-w-lg  flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
-        <p className="text-xl text-jet">OpenAI API Key</p>
-        <p className="text-lg leading-5 text-gray-500">
-          Before you can start using DocsGPT we need you to provide an API key
-          for llm. Currently, we support only OpenAI but soon many more. You can
-          find it here.
-        </p>
-        <input
-          type="text"
-          className="h-10 w-full border-b-2 border-jet focus:outline-none"
-          value={key}
-          maxLength={100}
-          placeholder="API Key"
-          onChange={(e) => setKey(e.target.value)}
-        />
-        <div className="flex justify-between">
-          {formError && (
-            <p className="text-sm text-red-500">Please enter a valid API key</p>
-          )}
-          <button
-            onClick={() => handleSubmit()}
-            className="ml-auto h-10 w-20 rounded-lg bg-violet-800 text-white transition-all hover:bg-violet-700"
-          >
-            Save
-          </button>
-        </div>
-      </article>
-    </div>
-  );
-}

frontend/src/components/Conversation/Conversation.tsx

@@ -1,25 +0,0 @@
-import { useMediaQuery } from '../../hooks';
-import { selectIsMenuOpen } from '../../store';
-import { useSelector } from 'react-redux';
-
-export default function Conversation() {
-  const isMobile = useMediaQuery('(max-width: 768px)');
-  const isMenuOpen = useSelector(selectIsMenuOpen);
-
-  return (
-    //Parent div for all content shown through App.tsx routing needs to have this styling.
-    <div
-      className={`${
-        isMobile
-          ? isMenuOpen
-            ? 'mt-80'
-            : 'mt-16'
-          : isMenuOpen
-          ? 'md:ml-72 lg:ml-96'
-          : 'ml-16'
-      } h-full w-full p-6 transition-all`}
-    >
-      Docs GPT Chat Placeholder
-    </div>
-  );
-}

frontend/src/components/Navigation.tsx

@@ -1,163 +0,0 @@
-import { useDispatch, useSelector } from 'react-redux';
-import { NavLink } from 'react-router-dom';
-import { useMediaQuery } from '../hooks';
-import {
-  toggleApiKeyModal,
-  selectIsMenuOpen,
-  toggleIsMenuOpen,
-} from '../store';
-import Arrow1 from '../imgs/arrow.svg';
-import Hamburger from '../imgs/hamburger.svg';
-import Key from '../imgs/key.svg';
-import Info from '../imgs/info.svg';
-import Link from '../imgs/link.svg';
-import Exit from '../imgs/exit.svg';
-
-//TODO - Need to replace Chat button to open secondary nav with scrollable past chats option and new chat at top
-//TODO - Need to add Discord and Github links
-
-function MobileNavigation({}) {
-  const dispatch = useDispatch();
-  const isMenuOpen = useSelector(selectIsMenuOpen);
-
-  return (
-    <div
-      className={`${
-        isMenuOpen ? 'border-b-2 border-gray-100' : 'h-16'
-      } fixed flex w-full flex-col bg-gray-50 transition-all`}
-    >
-      <div className="h-16 w-full border-b-2 border-gray-100">
-        {isMenuOpen ? (
-          <>
-            <button
-              className="mt-5 ml-6 h-6 w-6"
-              onClick={() => dispatch(toggleIsMenuOpen())}
-            >
-              <img src={Exit} alt="menu toggle" className="w-5" />
-            </button>
-          </>
-        ) : (
-          <>
-            <button
-              className="mt-5 ml-6 h-6 w-6"
-              onClick={() => dispatch(toggleIsMenuOpen())}
-            >
-              <img src={Hamburger} alt="menu toggle" className="w-7" />
-            </button>
-          </>
-        )}
-      </div>
-      {isMenuOpen && (
-        <nav className="my-4 flex flex-col">
-          <NavLink
-            to="/"
-            className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100"
-          >
-            <img src={Info} alt="info" className="ml-2 w-5" />
-            <p className="my-auto text-eerie-black">Chat</p>
-          </NavLink>
-          <NavLink
-            to="/about"
-            className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100"
-          >
-            <img src={Info} alt="info" className="ml-2 w-5" />
-            <p className="my-auto text-eerie-black">About</p>
-          </NavLink>
-          <div className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100">
-            <img src={Link} alt="info" className="ml-2 w-5" />
-            <p className="my-auto text-eerie-black">Discord</p>
-          </div>
-          <div className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100">
-            <img src={Link} alt="info" className="ml-2 w-5" />
-            <p className="my-auto text-eerie-black">Github</p>
-          </div>
-          <div
-            className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100"
-            onClick={() => dispatch(toggleApiKeyModal())}
-          >
-            <img src={Key} alt="info" className="ml-2 w-5" />
-            <p className="my-auto text-eerie-black">Reset Key</p>
-          </div>
-        </nav>
-      )}
-    </div>
-  );
-}
-
-function DesktopNavigation() {
-  const dispatch = useDispatch();
-  const isMenuOpen = useSelector(selectIsMenuOpen);
-
-  return (
-    <div
-      className={`${
-        isMenuOpen ? 'w-72 lg:w-96' : 'w-16'
-      } fixed flex h-screen flex-col border-r-2 border-gray-100 bg-gray-50 transition-all`}
-    >
-      <div
-        className={`${
-          isMenuOpen ? 'w-full' : 'w-16'
-        } ml-auto h-16 border-b-2 border-gray-100`}
-      >
-        <button
-          className="float-right mr-5 mt-5 h-5 w-5"
-          onClick={() => dispatch(toggleIsMenuOpen())}
-        >
-          <img
-            src={Arrow1}
-            alt="menu toggle"
-            className={`${
-              isMenuOpen ? 'rotate-0' : 'rotate-180'
-            } m-auto w-3 transition-all`}
-          />
-        </button>
-      </div>
-
-      {isMenuOpen && (
-        <>
-          <div className="flex-grow border-b-2 border-gray-100"></div>
-
-          <div className="flex h-16 flex-col border-b-2 border-gray-100">
-            <div
-              className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
-              onClick={() => dispatch(toggleApiKeyModal())}
-            >
-              <img src={Key} alt="key" className="ml-2 w-6" />
-              <p className="my-auto text-eerie-black">Reset Key</p>
-            </div>
-          </div>
-
-          <div className="flex h-48 flex-col border-b-2 border-gray-100">
-            <NavLink
-              to="/about"
-              className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
-            >
-              <img src={Info} alt="info" className="ml-2 w-5" />
-              <p className="my-auto text-eerie-black">About</p>
-            </NavLink>
-
-            <div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
-              <img src={Link} alt="link" className="ml-2 w-5" />
-              <p className="my-auto text-eerie-black">Discord</p>
-            </div>
-
-            <div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
-              <img src={Link} alt="link" className="ml-2 w-5" />
-              <p className="my-auto text-eerie-black">Github</p>
-            </div>
-          </div>
-        </>
-      )}
-    </div>
-  );
-}
-
-export default function Navigation() {
-  const isMobile = useMediaQuery('(max-width: 768px)');
-
-  if (isMobile) {
-    return <MobileNavigation />;
-  } else {
-    return <DesktopNavigation />;
-  }
-}

frontend/src/conversation/Conversation.tsx

@@ -0,0 +1,33 @@
+import { useEffect, useRef } from 'react';
+import { useSelector } from 'react-redux';
+import Hero from '../Hero';
+import ConversationBubble from './ConversationBubble';
+import ConversationInput from './ConversationInput';
+import { selectConversation } from './conversationSlice';
+
+export default function Conversation() {
+  const messages = useSelector(selectConversation);
+  const endMessageRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => endMessageRef?.current?.scrollIntoView());
+
+  return (
+    <div className="flex justify-center p-6">
+      <div className="w-10/12 transition-all md:w-1/2">
+        {messages.map((message, index) => {
+          return (
+            <ConversationBubble
+              ref={index === messages.length - 1 ? endMessageRef : null}
+              className="mb-7"
+              key={index}
+              message={message.text}
+              type={message.type}
+            ></ConversationBubble>
+          );
+        })}
+        {messages.length === 0 && <Hero className="mt-24"></Hero>}
+      </div>
+      <ConversationInput className="fixed bottom-2 w-10/12 md:w-[50%]"></ConversationInput>
+    </div>
+  );
+}

frontend/src/conversation/ConversationBubble.tsx

@@ -0,0 +1,26 @@
+import { forwardRef } from 'react';
+import Avatar from '../Avatar';
+import { MESSAGE_TYPE } from './conversationModels';
+
+const ConversationBubble = forwardRef<
+  HTMLDivElement,
+  {
+    message: string;
+    type: MESSAGE_TYPE;
+    className: string;
+  }
+>(function ConversationBubble({ message, type, className }, ref) {
+  return (
+    <div
+      ref={ref}
+      className={`flex rounded-3xl ${
+        type === 'QUESTION' ? '' : 'bg-gray-1000'
+      } py-7 px-5 ${className}`}
+    >
+      <Avatar avatar={type === 'QUESTION' ? '👤' : '🦖'}></Avatar>
+      <p className="ml-5">{message}</p>
+    </div>
+  );
+});
+
+export default ConversationBubble;

frontend/src/conversation/ConversationInput.tsx

@@ -0,0 +1,21 @@
+import Send from './../assets/send.svg';
+
+export default function ConversationInput({
+  className,
+}: {
+  className?: string;
+}) {
+  return (
+    <div className={`${className} flex`}>
+      <div
+        contentEditable
+        className={`min-h-5 border-000000 overflow-x-hidden; max-h-24 w-full overflow-y-auto rounded-xl border bg-white p-2 pr-9 opacity-100 focus:border-2 focus:outline-none`}
+      ></div>
+      <img
+        onClick={() => console.log('here')}
+        src={Send}
+        className="relative right-9"
+      ></img>
+    </div>
+  );
+}

frontend/src/conversation/conversationModels.ts

@@ -0,0 +1,10 @@
+export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER';
+
+export interface Message {
+  text: string;
+  type: MESSAGE_TYPE;
+}
+
+export interface ConversationState {
+  conversation: Message[];
+}

frontend/src/conversation/conversationSlice.ts

@@ -0,0 +1,50 @@
+import { createSlice, PayloadAction } from '@reduxjs/toolkit';
+import store from '../store';
+import { ConversationState, Message } from './conversationModels';
+
+// harcoding the initial state just for demo
+const initialState: ConversationState = {
+  conversation: [
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    { text: 'ChatGPT is large learning model', type: 'ANSWER' },
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    { text: 'ChatGPT is large learning model', type: 'ANSWER' },
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    { text: 'ChatGPT is large learning model', type: 'ANSWER' },
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    { text: 'ChatGPT is large learning model', type: 'ANSWER' },
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    { text: 'ChatGPT is large learning model', type: 'ANSWER' },
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    { text: 'ChatGPT is large learning model', type: 'ANSWER' },
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    { text: 'ChatGPT is large learning model', type: 'ANSWER' },
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    { text: 'ChatGPT is large learning model', type: 'ANSWER' },
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    { text: 'ChatGPT is large learning model', type: 'ANSWER' },
+    { text: 'what is ChatGPT', type: 'QUESTION' },
+    {
+      text: 'ChatGPT is large learning model',
+      type: 'ANSWER',
+    },
+  ],
+};
+
+export const conversationSlice = createSlice({
+  name: 'conversation',
+  initialState,
+  reducers: {
+    addMessage(state, action: PayloadAction<Message>) {
+      state.conversation.push(action.payload);
+    },
+  },
+});
+
+export const { addMessage } = conversationSlice.actions;
+
+type RootState = ReturnType<typeof store.getState>;
+export const selectConversation = (state: RootState) =>
+  state.conversation.conversation;
+
+export default conversationSlice.reducer;

frontend/src/hooks.ts

@@ -1,22 +0,0 @@
-import { useState, useEffect } from 'react';
-
-export function useMediaQuery(query: string): boolean {
-  const [matches, setMatches] = useState(false);
-
-  useEffect(() => {
-    const media = window.matchMedia(query);
-
-    if (media.matches !== matches) {
-      setMatches(media.matches);
-    }
-
-    const listener = () => {
-      setMatches(media.matches);
-    };
-
-    media.addEventListener('resize', listener);
-    return () => media.removeEventListener('resize', listener);
-  }, [matches, query]);
-
-  return matches;
-}

frontend/src/models/misc.ts

@@ -0,0 +1,5 @@
+export type ActiveState = 'ACTIVE' | 'INACTIVE';
+
+export type User = {
+  avatar: string;
+};

frontend/src/preferences/APIKeyModal.tsx

@@ -0,0 +1,83 @@
+import { useState } from 'react';
+import { useDispatch } from 'react-redux';
+import { ActiveState } from '../models/misc';
+import { setApiKey } from './preferenceSlice';
+
+export default function APIKeyModal({
+  modalState,
+  setModalState,
+  isCancellable = true,
+}: {
+  modalState: ActiveState;
+  setModalState: (val: ActiveState) => void;
+  isCancellable?: boolean;
+}) {
+  const dispatch = useDispatch();
+  const [key, setKey] = useState('');
+  const [isError, setIsError] = useState(false);
+
+  function handleSubmit() {
+    if (key.length <= 1) {
+      setIsError(true);
+    } else {
+      dispatch(setApiKey(key));
+      setModalState('INACTIVE');
+      setKey('');
+      setIsError(false);
+    }
+  }
+
+  function handleCancel() {
+    setKey('');
+    setIsError(false);
+    setModalState('INACTIVE');
+  }
+
+  return (
+    <div
+      className={`${
+        modalState === 'ACTIVE' ? 'visible' : 'hidden'
+      } absolute z-30  h-screen w-screen  bg-gray-alpha`}
+    >
+      <article className="mx-auto mt-24 flex w-[90vw] max-w-lg  flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
+        <p className="text-xl text-jet">OpenAI API Key</p>
+        <p className="text-lg leading-5 text-gray-500">
+          Before you can start using DocsGPT we need you to provide an API key
+          for llm. Currently, we support only OpenAI but soon many more. You can
+          find it here.
+        </p>
+        <input
+          type="text"
+          className="h-10 w-full border-b-2 border-jet focus:outline-none"
+          value={key}
+          maxLength={100}
+          placeholder="API Key"
+          onChange={(e) => setKey(e.target.value)}
+        />
+        <div className="flex flex-row-reverse">
+          <div>
+            <button
+              onClick={() => handleSubmit()}
+              className="ml-auto h-10 w-20 rounded-lg bg-violet-800 text-white transition-all hover:bg-violet-700"
+            >
+              Save
+            </button>
+            {isCancellable && (
+              <button
+                onClick={() => handleCancel()}
+                className="ml-5 h-10 w-20 rounded-lg border border-violet-700 bg-white text-violet-800 transition-all hover:bg-violet-700 hover:text-white"
+              >
+                Cancel
+              </button>
+            )}
+          </div>
+          {isError && (
+            <p className="mr-auto text-sm text-red-500">
+              Please enter a valid API key
+            </p>
+          )}
+        </div>
+      </article>
+    </div>
+  );
+}

frontend/src/preferences/preferenceSlice.ts

@@ -0,0 +1,29 @@
+import { createSlice } from '@reduxjs/toolkit';
+import store from '../store';
+
+interface Preference {
+  apiKey: string;
+}
+
+const initialState: Preference = {
+  apiKey: '',
+};
+
+export const prefSlice = createSlice({
+  name: 'preference',
+  initialState,
+  reducers: {
+    setApiKey: (state, action) => {
+      state.apiKey = action.payload;
+    },
+  },
+});
+
+export const { setApiKey } = prefSlice.actions;
+export default prefSlice.reducer;
+
+type RootState = ReturnType<typeof store.getState>;
+
+export const selectApiKey = (state: RootState) => state.preference.apiKey;
+export const selectApiKeyStatus = (state: RootState) =>
+  !!state.preference.apiKey;

frontend/src/store.ts

@@ -1,48 +1,12 @@
-import { configureStore, createSlice, PayloadAction } from '@reduxjs/toolkit';
-
-interface State {
-  isApiKeyModalOpen: boolean;
-  apiKey: string;
-  isMenuOpen: boolean;
-}
-
-const initialState: State = {
-  isApiKeyModalOpen: false,
-  apiKey: '',
-  isMenuOpen: false,
-};
-
-export const slice = createSlice({
-  name: 'app',
-  initialState,
-  reducers: {
-    toggleApiKeyModal: (state) => {
-      state.isApiKeyModalOpen = !state.isApiKeyModalOpen;
-      console.log('showApiKeyModal', state.isApiKeyModalOpen);
-    },
-    setApiKey: (state, action: PayloadAction<string>) => {
-      state.apiKey = action.payload;
-      console.log('setApiKey', action.payload);
-    },
-    toggleIsMenuOpen: (state) => {
-      state.isMenuOpen = !state.isMenuOpen;
-    },
-  },
-});
-
-export const { toggleApiKeyModal, setApiKey, toggleIsMenuOpen } = slice.actions;
-
-const store = configureStore({
-  reducer: {
-    app: slice.reducer,
-  },
-});
-
-type RootState = ReturnType<typeof store.getState>;
-
-export const selectIsApiKeyModalOpen = (state: RootState) =>
-  state.app.isApiKeyModalOpen;
-export const selectApiKey = (state: RootState) => state.app.apiKey;
-export const selectIsMenuOpen = (state: RootState) => state.app.isMenuOpen;
-
-export default store;
+import { configureStore } from '@reduxjs/toolkit';
+import { conversationSlice } from './conversation/conversationSlice';
+import { prefSlice } from './preferences/preferenceSlice';
+
+const store = configureStore({
+  reducer: {
+    preference: prefSlice.reducer,
+    conversation: conversationSlice.reducer,
+  },
+});
+
+export default store;

frontend/tailwind.config.cjs

@@ -11,6 +11,7 @@ module.exports = {
         'eerie-black': '#212121',
         jet: '#343541',
         'gray-alpha': 'rgba(0,0,0, .1)',
+        'gray-1000': '#F6F6F6',
       },
     },
   },

scripts/ingest.py

@@ -1,3 +1,5 @@
+from collections import defaultdict
+import os
 import sys
 import nltk
 import dotenv
@@ -18,13 +20,17 @@ app = typer.Typer(add_completion=False)
 nltk.download('punkt', quiet=True)
 nltk.download('averaged_perceptron_tagger', quiet=True)
 
+
 #Splits all files in specified folder to documents
 @app.command()
-def ingest(directory: Optional[str] = typer.Option("inputs",
-                                                   help="Path to the directory for index creation."),
-           files: Optional[List[str]] = typer.Option(None,
-                                                   help="""File paths to use (Optional; overrides directory).
-                                                        E.g. --files inputs/1.md --files inputs/2.md"""),
+def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
+                                                   help="Whether to skip price confirmation"),
+           dir: Optional[List[str]] = typer.Option(["inputs"],
+                                                   help="""List of paths to directory for index creation.
+                                                        E.g. --dir inputs --dir inputs2"""),
+           file: Optional[List[str]] = typer.Option(None,
+                                                   help="""File paths to use (Optional; overrides dir).
+                                                        E.g. --file inputs/1.md --file inputs/2.md"""),
            recursive: Optional[bool] = typer.Option(True,
                                                    help="Whether to recursively search in subdirectories."),
            limit: Optional[int] = typer.Option(None,
@@ -38,27 +44,39 @@ def ingest(directory: Optional[str] = typer.Option("inputs",
         Creates index from specified location or files.
         By default /inputs folder is used, .rst and .md are parsed.
     """
-    raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=files, recursive=recursive,
-                                     required_exts=formats, num_files_limit=limit,
-                                     exclude_hidden=exclude).load_data()
-    raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
-    print(raw_docs)
-    # Here we split the documents, as needed, into smaller chunks.
-    # We do this due to the context limits of the LLMs.
-    text_splitter = RecursiveCharacterTextSplitter()
-    docs = text_splitter.split_documents(raw_docs)
 
-    # Here we check for command line arguments for bot calls.
-    # If no argument exists or the permission_bypass_flag argument is not '-y',
-    # user permission is requested to call the API.
-    if len(sys.argv) > 1:
-        permission_bypass_flag = sys.argv[1]
-        if permission_bypass_flag == '-y':
-            call_openai_api(docs)
+    def process_one_docs(directory, folder_name):
+        raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=file, recursive=recursive,
+                                         required_exts=formats, num_files_limit=limit,
+                                         exclude_hidden=exclude).load_data()
+        raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+        # Here we split the documents, as needed, into smaller chunks.
+        # We do this due to the context limits of the LLMs.
+        text_splitter = RecursiveCharacterTextSplitter()
+        docs = text_splitter.split_documents(raw_docs)
+
+        # Here we check for command line arguments for bot calls.
+        # If no argument exists or the yes is not True, then the
+        # user permission is requested to call the API.
+        if len(sys.argv) > 1:
+            if yes:
+                call_openai_api(docs, folder_name)
+            else:
+                get_user_permission(docs, folder_name)
         else:
-            get_user_permission(docs)
-    else:
-        get_user_permission(docs)
+            get_user_permission(docs, folder_name)
+
+    folder_counts = defaultdict(int)
+    folder_names = []
+    for dir_path in dir:
+        folder_name = os.path.basename(os.path.normpath(dir_path))
+        folder_counts[folder_name] += 1
+        if folder_counts[folder_name] > 1:
+            folder_name = f"{folder_name}_{folder_counts[folder_name]}"
+        folder_names.append(folder_name)
+
+    for directory, folder_name in zip(dir, folder_names):
+        process_one_docs(directory, folder_name)
 
 if __name__ == "__main__":
   app()

scripts/parser/open_ai_func.py

@@ -1,9 +1,17 @@
+import os
 import faiss
 import pickle
 import tiktoken
 from langchain.vectorstores import FAISS
 from langchain.embeddings import OpenAIEmbeddings
 
+#from langchain.embeddings import HuggingFaceEmbeddings
+#from langchain.embeddings import HuggingFaceInstructEmbeddings
+#from langchain.embeddings import CohereEmbeddings
+
+from retry import retry
+
+
 
 def num_tokens_from_string(string: str, encoding_name: str) -> int:
 # Function to convert string to tokens and estimate user cost.
@@ -12,8 +20,17 @@ def num_tokens_from_string(string: str, encoding_name: str) -> int:
     total_price = ((num_tokens/1000) * 0.0004)
     return num_tokens, total_price
 
-def call_openai_api(docs):
+@retry(tries=10, delay=60)
+def store_add_texts_with_retry(store, i):
+    store.add_texts([i.page_content], metadatas=[i.metadata])
+
+def call_openai_api(docs, folder_name):
 # Function to create a vector store from the documents and save it to disk.
+
+    # create output folder if it doesn't exist
+    if not os.path.exists(f"outputs/{folder_name}"):
+        os.makedirs(f"outputs/{folder_name}")
+
     from tqdm import tqdm
     docs_test = [docs[0]]
     # remove the first element from docs
@@ -22,34 +39,25 @@ def call_openai_api(docs):
     #docs = docs[:n]
     c1 = 0
     store = FAISS.from_documents(docs_test, OpenAIEmbeddings())
+
+    # Uncomment for MPNet embeddings
+    # model_name = "sentence-transformers/all-mpnet-base-v2"
+    # hf = HuggingFaceEmbeddings(model_name=model_name)
+    # store = FAISS.from_documents(docs_test, hf)
     for i in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs), bar_format='{l_bar}{bar}| Time Left: {remaining}'):
         try:
-            import time
-            store.add_texts([i.page_content], metadatas=[i.metadata])
+            store_add_texts_with_retry(store, i)
         except Exception as e:
             print(e)
             print("Error on ", i)
             print("Saving progress")
             print(f"stopped at {c1} out of {len(docs)}")
-            faiss.write_index(store.index, "docs.index")
-            store_index_bak = store.index
-            store.index = None
-            with open("faiss_store.pkl", "wb") as f:
-                pickle.dump(store, f)
-            print("Sleeping for 60 seconds and trying again")
-            time.sleep(60)
-            faiss.write_index(store_index_bak, "docs.index")
-            store.index = store_index_bak
-            store.add_texts([i.page_content], metadatas=[i.metadata])
+            store.save_local(f"outputs/{folder_name}")
+            break
         c1 += 1
+    store.save_local(f"outputs/{folder_name}")
 
-
-    faiss.write_index(store.index, "docs.index")
-    store.index = None
-    with open("faiss_store.pkl", "wb") as f:
-        pickle.dump(store, f)
-
-def get_user_permission(docs):
+def get_user_permission(docs, folder_name):
 # Function to ask user permission to call the OpenAI api and spend their OpenAI funds.
     # Here we convert the docs list to a string and calculate the number of OpenAI tokens the string represents.
     #docs_content = (" ".join(docs))
@@ -65,8 +73,8 @@ def get_user_permission(docs):
     #Here we check for user permission before calling the API.
     user_input = input("Price Okay? (Y/N) \n").lower()
     if user_input == "y":
-        call_openai_api(docs)
+        call_openai_api(docs, folder_name)
     elif user_input == "":
-        call_openai_api(docs)
+        call_openai_api(docs, folder_name)
     else:
         print("The API was not called. No money was spent.")