diff --git a/.gitignore b/.gitignore index 26e76270..7b790790 100644 --- a/.gitignore +++ b/.gitignore @@ -131,4 +131,6 @@ dmypy.json .idea/ # macOS -.DS_Store \ No newline at end of file +.DS_Store + +application/vectors/ \ No newline at end of file diff --git a/application/app.py b/application/app.py index 2b99d612..847f1895 100644 --- a/application/app.py +++ b/application/app.py @@ -8,6 +8,7 @@ import faiss from langchain import OpenAI from langchain.chains import VectorDBQAWithSourcesChain from langchain.prompts import PromptTemplate +import requests # Redirect PosixPath to WindowsPath on Windows import platform @@ -40,11 +41,13 @@ def api_answer(): api_key = data["api_key"] # check if the vectorstore is set if "active_docs" in data: - vectorstore = "vectorstores/" + data["active_docs"] + vectorstore = "vectors/" + data["active_docs"] + if data['active_docs'] == "default": + vectorstore = "" else: vectorstore = "" - print(vectorstore) + # loading the index and the store and the prompt template index = faiss.read_index(f"{vectorstore}docs.index") @@ -73,6 +76,30 @@ def api_answer(): return result +@app.route("/api/docs_check", methods=["POST"]) +def check_docs(): + # check if docs exist in a vectorstore folder + data = request.get_json() + vectorstore = "vectors/" + data["docs"] + base_path = 'https://raw.githubusercontent.com/arc53/DocsHUB/main/' + # + if os.path.exists(vectorstore): + return {"status": 'exists'} + else: + r = requests.get(base_path + vectorstore + "docs.index") + # save to vectors directory + # check if the directory exists + if not os.path.exists(vectorstore): + os.makedirs(vectorstore) + + with open(vectorstore + "docs.index", "wb") as f: + f.write(r.content) + # download the store + r = requests.get(base_path + vectorstore + "faiss_store.pkl") + with open(vectorstore + "faiss_store.pkl", "wb") as f: + f.write(r.content) + + return {"status": 'loaded'} # handling CORS @app.after_request diff --git a/application/static/dist/css/output.css b/application/static/dist/css/output.css index 9d1609a7..7a6e59a4 100644 --- a/application/static/dist/css/output.css +++ b/application/static/dist/css/output.css @@ -680,6 +680,11 @@ video { background-color: rgb(59 130 246 / var(--tw-bg-opacity)); } +.bg-gray-50 { + --tw-bg-opacity: 1; + background-color: rgb(249 250 251 / var(--tw-bg-opacity)); +} + .bg-gray-900 { --tw-bg-opacity: 1; background-color: rgb(17 24 39 / var(--tw-bg-opacity)); @@ -695,11 +700,6 @@ video { background-color: rgb(229 231 235 / var(--tw-bg-opacity)); } -.bg-gray-50 { - --tw-bg-opacity: 1; - background-color: rgb(249 250 251 / var(--tw-bg-opacity)); -} - .p-2 { padding: 0.5rem; } @@ -785,6 +785,16 @@ video { color: rgb(17 24 39 / var(--tw-text-opacity)); } +.text-green-500 { + --tw-text-opacity: 1; + color: rgb(34 197 94 / var(--tw-text-opacity)); +} + +.text-red-500 { + --tw-text-opacity: 1; + color: rgb(239 68 68 / var(--tw-text-opacity)); +} + .opacity-75 { opacity: 0.75; } @@ -867,43 +877,6 @@ video { --tw-ring-color: rgb(59 130 246 / var(--tw-ring-opacity)); } -@media (prefers-color-scheme: dark) { - .dark\:border-gray-600 { - --tw-border-opacity: 1; - border-color: rgb(75 85 99 / var(--tw-border-opacity)); - } - - .dark\:bg-gray-700 { - --tw-bg-opacity: 1; - background-color: rgb(55 65 81 / var(--tw-bg-opacity)); - } - - .dark\:text-white { - --tw-text-opacity: 1; - color: rgb(255 255 255 / var(--tw-text-opacity)); - } - - .dark\:placeholder-gray-400::-moz-placeholder { - --tw-placeholder-opacity: 1; - color: rgb(156 163 175 / var(--tw-placeholder-opacity)); - } - - .dark\:placeholder-gray-400::placeholder { - --tw-placeholder-opacity: 1; - color: rgb(156 163 175 / var(--tw-placeholder-opacity)); - } - - .dark\:focus\:border-blue-500:focus { - --tw-border-opacity: 1; - border-color: rgb(59 130 246 / var(--tw-border-opacity)); - } - - .dark\:focus\:ring-blue-500:focus { - --tw-ring-opacity: 1; - --tw-ring-color: rgb(59 130 246 / var(--tw-ring-opacity)); - } -} - @media (min-width: 640px) { .sm\:my-8 { margin-top: 2rem; diff --git a/application/static/src/choiceChange.js b/application/static/src/choiceChange.js index 24887723..9992ac34 100644 --- a/application/static/src/choiceChange.js +++ b/application/static/src/choiceChange.js @@ -1,3 +1,29 @@ - document.getElementById("select-docs").addEventListener("change", function() { - localStorage.setItem('activeDocs', this.value) - }); \ No newline at end of file + +function docsIndex() { + // loads latest index from https://raw.githubusercontent.com/arc53/DocsHUB/main/combined.json + // and stores it in localStorage + fetch('https://raw.githubusercontent.com/arc53/DocsHUB/main/combined.json') + .then(response => response.json()) + .then(data => { + console.log('Success:', data); + localStorage.setItem("docsIndex", JSON.stringify(data)); + } + ) + +} + +document.getElementById("select-docs").addEventListener("change", function() { +localStorage.setItem('activeDocs', this.value) + fetch('/api/docs_check', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({docs: this.value}), + }).then(response => response.json()).then( + data => { + console.log('Success:', data); + } + ) +}); + diff --git a/application/templates/index.html b/application/templates/index.html index 50becfca..2e7b0ce8 100644 --- a/application/templates/index.html +++ b/application/templates/index.html @@ -61,12 +61,10 @@ This will return a new DataFrame with all the columns from both tables, and only

The source code is available on Github


Currently It uses python pandas documentation, so it will respond to information relevant to pandas. If you want to train it on different documentation - please follow this guide


If you want to launch it on your own server - follow this guide


- + @@ -103,6 +101,35 @@ This will return a new DataFrame with all the columns from both tables, and only console.log("apiKey is not set") document.getElementById('modal').classList.toggle('hidden') } + if (localStorage.getItem('docsIndex') === null) { + console.log("docsIndex is not set") + docsIndex() + } + + docsIndex = localStorage.getItem('docsIndex') + // create option on select with id select-docs + var select = document.getElementById("select-docs"); + // convert docsIndex to json + docsIndex = JSON.parse(docsIndex) + // create option for each key in docsIndex + for (var key in docsIndex) { + var option = document.createElement("option"); + console.log(key) + console.log(docsIndex[key]) + if (docsIndex[key].name == docsIndex[key].language) { + option.text = docsIndex[key].name + " " + docsIndex[key].version; + option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/"; + select.add(option); + } + else { + option.text = docsIndex[key].name + " " + docsIndex[key].version; + option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/"; + select.add(option); + } + } + + + diff --git a/application/vectorstores/ethereum/solidity/docs.index b/application/vectorstores/ethereum/solidity/docs.index deleted file mode 100644 index dbde072e..00000000 Binary files a/application/vectorstores/ethereum/solidity/docs.index and /dev/null differ diff --git a/application/vectorstores/ethereum/solidity/faiss_store.pkl b/application/vectorstores/ethereum/solidity/faiss_store.pkl deleted file mode 100644 index 9e3276a5..00000000 Binary files a/application/vectorstores/ethereum/solidity/faiss_store.pkl and /dev/null differ diff --git a/application/vectorstores/python/pandas/docs.index b/application/vectorstores/python/pandas/docs.index deleted file mode 100644 index b5e33008..00000000 Binary files a/application/vectorstores/python/pandas/docs.index and /dev/null differ diff --git a/application/vectorstores/python/pandas/faiss_store.pkl b/application/vectorstores/python/pandas/faiss_store.pkl deleted file mode 100644 index bef85b3e..00000000 Binary files a/application/vectorstores/python/pandas/faiss_store.pkl and /dev/null differ diff --git a/application/vectorstores/python/scikit-learn/docs.index b/application/vectorstores/python/scikit-learn/docs.index deleted file mode 100644 index 7461d041..00000000 Binary files a/application/vectorstores/python/scikit-learn/docs.index and /dev/null differ diff --git a/application/vectorstores/python/scikit-learn/faiss_store.pkl b/application/vectorstores/python/scikit-learn/faiss_store.pkl deleted file mode 100644 index c2b51849..00000000 Binary files a/application/vectorstores/python/scikit-learn/faiss_store.pkl and /dev/null differ