Merge branch 'main' into main

.gitignore

@@ -158,3 +158,6 @@ frontend/*.ntvs*
 frontend/*.njsproj
 frontend/*.sln
 frontend/*.sw?
+
+application/vectors/
+

README.md

@@ -34,11 +34,11 @@ You can find our [Roadmap](https://github.com/orgs/arc53/projects/2) here, pleas
 
 
 ## Project structure
-application - flask app (main application)
+- Application - flask app (main application)
 
-extensions - chrome extension
+- Extensions - chrome extension
 
-scripts - script that creates similarity search index and store for other libraries 
+- Scripts - script that creates similarity search index and store for other libraries. 
 
 ## QuickStart
 Please note: current vector database uses pandas Python documentation, thus responses will be related to it, if you want to use other docs please follow a guide below

application/app.py

@@ -8,6 +8,7 @@ import faiss
 from langchain import OpenAI
 from langchain.chains import VectorDBQAWithSourcesChain
 from langchain.prompts import PromptTemplate
+import requests
 
 # Redirect PosixPath to WindowsPath on Windows
 import platform
@@ -23,6 +24,13 @@ dotenv.load_dotenv()
 with open("combine_prompt.txt", "r") as f:
     template = f.read()
 
+# check if OPENAI_API_KEY is set
+if os.getenv("OPENAI_API_KEY") is not None:
+    api_key_set = True
+
+else:
+    api_key_set = False
+
 
 
 app = Flask(__name__)
@@ -30,26 +38,29 @@ app = Flask(__name__)
 
 @app.route("/")
 def home():
-    return render_template("index.html")
+    return render_template("index.html", api_key_set=api_key_set)
 
 
 @app.route("/api/answer", methods=["POST"])
 def api_answer():
     data = request.get_json()
     question = data["question"]
-    api_key = data["api_key"]
+    if not api_key_set:
+        api_key = data["api_key"]
+    else:
+        api_key = os.getenv("OPENAI_API_KEY")
+
     # check if the vectorstore is set
     if "active_docs" in data:
-        vectorstore = "vectorstores/" + data["active_docs"]
-
+        vectorstore = "vectors/" + data["active_docs"]
+        if data['active_docs'] == "default":
+            vectorstore = ""
     else:
         vectorstore = ""
-    print(vectorstore)
 
     # loading the index and the store and the prompt template
     index = faiss.read_index(f"{vectorstore}docs.index")
 
-
     with open(f"{vectorstore}faiss_store.pkl", "rb") as f:
         store = pickle.load(f)
 
@@ -57,6 +68,7 @@ def api_answer():
     # create a prompt template
     c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template)
     # create a chain with the prompt template and the store
+
     chain = VectorDBQAWithSourcesChain.from_llm(llm=OpenAI(openai_api_key=api_key, temperature=0), vectorstore=store, combine_prompt=c_prompt)
     # fetch the answer
     result = chain({"question": question})
@@ -69,10 +81,32 @@ def api_answer():
     #     "answer": "The answer is 42",
     #     "sources": ["https://en.wikipedia.org/wiki/42_(number)", "https://en.wikipedia.org/wiki/42_(number)"]
     # }
-
-
     return result
 
+@app.route("/api/docs_check", methods=["POST"])
+def check_docs():
+    # check if docs exist in a vectorstore folder
+    data = request.get_json()
+    vectorstore = "vectors/" + data["docs"]
+    base_path = 'https://raw.githubusercontent.com/arc53/DocsHUB/main/'
+    #
+    if os.path.exists(vectorstore):
+        return {"status": 'exists'}
+    else:
+        r = requests.get(base_path + vectorstore + "docs.index")
+        # save to vectors directory
+        # check if the directory exists
+        if not os.path.exists(vectorstore):
+            os.makedirs(vectorstore)
+
+        with open(vectorstore + "docs.index", "wb") as f:
+            f.write(r.content)
+        # download the store
+        r = requests.get(base_path + vectorstore + "faiss_store.pkl")
+        with open(vectorstore + "faiss_store.pkl", "wb") as f:
+            f.write(r.content)
+
+        return {"status":  'loaded'}
 
 # handling CORS
 @app.after_request

application/static/dist/css/output.css

@@ -680,6 +680,11 @@ video {
   background-color: rgb(59 130 246 / var(--tw-bg-opacity));
 }
 
+.bg-gray-50 {
+  --tw-bg-opacity: 1;
+  background-color: rgb(249 250 251 / var(--tw-bg-opacity));
+}
+
 .bg-gray-900 {
   --tw-bg-opacity: 1;
   background-color: rgb(17 24 39 / var(--tw-bg-opacity));
@@ -695,11 +700,6 @@ video {
   background-color: rgb(229 231 235 / var(--tw-bg-opacity));
 }
 
-.bg-gray-50 {
-  --tw-bg-opacity: 1;
-  background-color: rgb(249 250 251 / var(--tw-bg-opacity));
-}
-
 .p-2 {
   padding: 0.5rem;
 }
@@ -867,43 +867,6 @@ video {
   --tw-ring-color: rgb(59 130 246 / var(--tw-ring-opacity));
 }
 
-@media (prefers-color-scheme: dark) {
-  .dark\:border-gray-600 {
-    --tw-border-opacity: 1;
-    border-color: rgb(75 85 99 / var(--tw-border-opacity));
-  }
-
-  .dark\:bg-gray-700 {
-    --tw-bg-opacity: 1;
-    background-color: rgb(55 65 81 / var(--tw-bg-opacity));
-  }
-
-  .dark\:text-white {
-    --tw-text-opacity: 1;
-    color: rgb(255 255 255 / var(--tw-text-opacity));
-  }
-
-  .dark\:placeholder-gray-400::-moz-placeholder {
-    --tw-placeholder-opacity: 1;
-    color: rgb(156 163 175 / var(--tw-placeholder-opacity));
-  }
-
-  .dark\:placeholder-gray-400::placeholder {
-    --tw-placeholder-opacity: 1;
-    color: rgb(156 163 175 / var(--tw-placeholder-opacity));
-  }
-
-  .dark\:focus\:border-blue-500:focus {
-    --tw-border-opacity: 1;
-    border-color: rgb(59 130 246 / var(--tw-border-opacity));
-  }
-
-  .dark\:focus\:ring-blue-500:focus {
-    --tw-ring-opacity: 1;
-    --tw-ring-color: rgb(59 130 246 / var(--tw-ring-opacity));
-  }
-}
-
 @media (min-width: 640px) {
   .sm\:my-8 {
     margin-top: 2rem;

application/static/favicon/site.webmanifest

@@ -0,0 +1 @@
+{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"}

application/static/src/authapi.js

@@ -1,15 +1,19 @@
 function resetApiKey() {
-  document.getElementById('modal').classList.toggle('hidden')
+  const modal = document.getElementById("modal");
+  modal.classList.toggle("hidden");
 }
 
+const apiKeyForm = document.getElementById("api-key-form");
+if (apiKeyForm) {
+  apiKeyForm.addEventListener("submit", function(event) {
+    event.preventDefault();
 
-var el2 = document.getElementById('api-key-form');
-if (el2) {
-    el2.addEventListener("submit", function (event) {
-        event.preventDefault()
-        var apiKey = document.getElementById("api-key-input").value;
-        document.getElementById('modal').classList.toggle('hidden')
-        localStorage.setItem('apiKey', apiKey)
-        document.getElementById('api-key-input').value = ''
-    });
+    const apiKeyInput = document.getElementById("api-key-input");
+    const apiKey = apiKeyInput.value;
+
+    localStorage.setItem("apiKey", apiKey);
+
+    apiKeyInput.value = "";
+    modal.classList.toggle("hidden");
+  });
 }

application/static/src/chat.js

@@ -13,6 +13,9 @@ if (el) {
         document.getElementById("message-input").value = "";
         document.getElementById("button-submit").innerHTML = '<i class="fa fa-circle-o-notch fa-spin"></i> Thinking...';
         document.getElementById("button-submit").disabled = true;
+        if (localStorage.getItem('activeDocs') == null) {
+            localStorage.setItem('activeDocs', 'default')
+        }
 
         fetch('/api/answer', {
             method: 'POST',
@@ -22,7 +25,7 @@ if (el) {
 
             body: JSON.stringify({question: message,
                 api_key: localStorage.getItem('apiKey'),
-                active_docs: localStorage.getItem('activeDocs'),}),
+                active_docs: localStorage.getItem('activeDocs')}),
         })
             .then(response => response.json())
             .then(data => {

application/static/src/choiceChange.js

@@ -1,3 +1,15 @@
-  document.getElementById("select-docs").addEventListener("change", function() {
-    localStorage.setItem('activeDocs', this.value)
-  });
+document.getElementById("select-docs").addEventListener("change", function() {
+localStorage.setItem('activeDocs', this.value)
+     fetch('/api/docs_check', {
+         method: 'POST',
+         headers: {
+             'Content-Type': 'application/json',
+         },
+         body: JSON.stringify({docs: this.value}),
+     }).then(response => response.json()).then(
+            data => {
+                console.log('Success:', data);
+            }
+     )
+});
+

application/templates/index.html

@@ -3,6 +3,11 @@
   <head>
     <title>DocsGPT 🦖 Preview</title>
     <link href="{{url_for('static',filename='dist/css/output.css')}}" rel="stylesheet">
+      <link rel="favicon" href="{{ url_for('static', filename='favicon/favicon.ico') }}">
+      <link rel="apple-touch-icon" sizes="180x180" href="{{ url_for('static', filename='favicon/apple-touch-icon.png') }}">
+    <link rel="icon" type="image/png" sizes="32x32" href="{{ url_for('static', filename='favicon/favicon-32x32.png') }}">
+    <link rel="icon" type="image/png" sizes="16x16" href="{{ url_for('static', filename='favicon/favicon-16x16.png') }}">
+    <link rel="manifest" href="{{ url_for('static', filename='favicon//site.webmanifest') }}">
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
 
 
@@ -18,7 +23,9 @@
       <h1 class="text-lg font-medium">DocsGPT 🦖 Preview</h1>
         <div>
       <a href="https://github.com/arc53/docsgpt" class="text-blue-500 hover:text-blue-800 text-sm">About</a>
+            {% if not api_key_set %}
       <button class="text-sm text-yellow-500 hover:text-yellow-800" onclick="resetApiKey()">Reset Key</button>
+        {% endif %}
             </div>
     </header>
     <div class="lg:flex ml-2 mr-2">
@@ -61,12 +68,10 @@ This will return a new DataFrame with all the columns from both tables, and only
           <p class="text-sm">The source code is available on <a href="https://github.com/arc53/docsgpt" class="text-blue-500 hover:text-blue-800">Github</a></p><br>
           <p class="text-sm">Currently It uses python pandas documentation, so it will respond to information relevant to pandas. If you want to train it on different documentation -  <a href="https://github.com/arc53/docsgpt/wiki/How-to-train-on-other-documentation" class="text-blue-500 hover:text-blue-800"> please follow this guide </a></p><br>
           <p class="text-sm">If you want to launch it on your own server - <a href="https://github.com/arc53/docsgpt/wiki/How-to-train-on-other-documentation" class="text-blue-500 hover:text-blue-800"> follow this guide </a></p><br>
-            <label  class="block mb-2 text-sm font-medium text-gray-900">Select pre-loaded documentation</label>
+            <label  class="block mb-2 text-sm font-medium text-gray-900">Select documentation from DocsHUB</label>
             <select id="select-docs" class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5">
-              <option selected>Choose documentation</option>
-              <option value="ethereum/solidity/">Solidity</option>
-              <option value="python/pandas/">Pandas</option>
-              <option value="python/scikit-learn/">scikit-learn</option>
+              <option selected value="default">Choose documentation</option>
+              <option value="default">Default</option>
             </select>
         </div>
     </div>
@@ -74,6 +79,8 @@ This will return a new DataFrame with all the columns from both tables, and only
   <div class="flex items-center justify-center h-full">
 
 </div>
+
+{% if not api_key_set %}
 <div class="fixed z-10 overflow-y-auto top-0 w-full left-0 hidden" id="modal">
   <div class="flex items-center justify-center min-height-100vh pt-4 px-4 pb-20 text-center sm:block sm:p-0">
     <div class="fixed inset-0 transition-opacity">
@@ -97,14 +104,65 @@ This will return a new DataFrame with all the columns from both tables, and only
     </div>
   </div>
 </div>
+{% endif %}
       <script>
-      // check if api_key is set
+          function docsIndex() {
+                // loads latest index from https://raw.githubusercontent.com/arc53/DocsHUB/main/combined.json
+                // and stores it in localStorage
+                fetch('https://d3dg1063dc54p9.cloudfront.net/combined.json')
+                    .then(response => response.json())
+                    .then(data => {
+                        localStorage.setItem("docsIndex", JSON.stringify(data));
+                        localStorage.setItem("docsIndexDate", Date.now());
+                        generateOptions()
+                    }
+
+                )
+
+            }
+          function generateOptions(){
+                docsIndex = localStorage.getItem('docsIndex')
+                // create option on select with id select-docs
+                var select = document.getElementById("select-docs");
+                // convert docsIndex to json
+                docsIndex = JSON.parse(docsIndex)
+                // create option for each key in docsIndex
+                for (var key in docsIndex) {
+                    var option = document.createElement("option");
+                    if (docsIndex[key].name == docsIndex[key].language) {
+                        option.text = docsIndex[key].name + " " + docsIndex[key].version;
+                        option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/";
+                        select.add(option);
+                    }
+                    else {
+                        option.text = docsIndex[key].name + " " + docsIndex[key].version;
+                        option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/";
+                        select.add(option);
+                    }
+                }
+
+          }
+        {% if not api_key_set %}
         if (localStorage.getItem('apiKey') === null) {
             console.log("apiKey is not set")
             document.getElementById('modal').classList.toggle('hidden')
         }
+        {% endif %}
+        if (localStorage.getItem('docsIndex') === null) {
+            console.log("docsIndex is not set")
+            docsIndex()
+        }
+        else if (localStorage.getItem("docsIndexDate") < Date.now() - 900000) {
+            console.log("docsIndex is older than 15 minutes")
+            docsIndex()
+        }
+
+        generateOptions()
+
   </script>
+    {% if not api_key_set %}
     <script src="{{url_for('static',filename='src/authapi.js')}}"></script>
+    {% endif %}
   <script src="{{url_for('static',filename='src/chat.js')}}"></script>
   <script src="{{url_for('static',filename='src/choiceChange.js')}}"></script>