diff --git a/.gitignore b/.gitignore index 10d6da0b..8b394e9b 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ frontend/*.ntvs* frontend/*.njsproj frontend/*.sln frontend/*.sw? + +application/vectors/ + diff --git a/README.md b/README.md index d17605b1..aefc72ac 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,11 @@ You can find our [Roadmap](https://github.com/orgs/arc53/projects/2) here, pleas ## Project structure -application - flask app (main application) +- Application - flask app (main application) -extensions - chrome extension +- Extensions - chrome extension -scripts - script that creates similarity search index and store for other libraries +- Scripts - script that creates similarity search index and store for other libraries. ## QuickStart Please note: current vector database uses pandas Python documentation, thus responses will be related to it, if you want to use other docs please follow a guide below diff --git a/application/app.py b/application/app.py index 2b99d612..aa9089ed 100644 --- a/application/app.py +++ b/application/app.py @@ -8,6 +8,7 @@ import faiss from langchain import OpenAI from langchain.chains import VectorDBQAWithSourcesChain from langchain.prompts import PromptTemplate +import requests # Redirect PosixPath to WindowsPath on Windows import platform @@ -23,6 +24,13 @@ dotenv.load_dotenv() with open("combine_prompt.txt", "r") as f: template = f.read() +# check if OPENAI_API_KEY is set +if os.getenv("OPENAI_API_KEY") is not None: + api_key_set = True + +else: + api_key_set = False + app = Flask(__name__) @@ -30,26 +38,29 @@ app = Flask(__name__) @app.route("/") def home(): - return render_template("index.html") + return render_template("index.html", api_key_set=api_key_set) @app.route("/api/answer", methods=["POST"]) def api_answer(): data = request.get_json() question = data["question"] - api_key = data["api_key"] + if not api_key_set: + api_key = data["api_key"] + else: + api_key = os.getenv("OPENAI_API_KEY") + # check if the vectorstore is set if "active_docs" in data: - vectorstore = "vectorstores/" + data["active_docs"] - + vectorstore = "vectors/" + data["active_docs"] + if data['active_docs'] == "default": + vectorstore = "" else: vectorstore = "" - print(vectorstore) # loading the index and the store and the prompt template index = faiss.read_index(f"{vectorstore}docs.index") - with open(f"{vectorstore}faiss_store.pkl", "rb") as f: store = pickle.load(f) @@ -57,6 +68,7 @@ def api_answer(): # create a prompt template c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template) # create a chain with the prompt template and the store + chain = VectorDBQAWithSourcesChain.from_llm(llm=OpenAI(openai_api_key=api_key, temperature=0), vectorstore=store, combine_prompt=c_prompt) # fetch the answer result = chain({"question": question}) @@ -69,10 +81,32 @@ def api_answer(): # "answer": "The answer is 42", # "sources": ["https://en.wikipedia.org/wiki/42_(number)", "https://en.wikipedia.org/wiki/42_(number)"] # } - - return result +@app.route("/api/docs_check", methods=["POST"]) +def check_docs(): + # check if docs exist in a vectorstore folder + data = request.get_json() + vectorstore = "vectors/" + data["docs"] + base_path = 'https://raw.githubusercontent.com/arc53/DocsHUB/main/' + # + if os.path.exists(vectorstore): + return {"status": 'exists'} + else: + r = requests.get(base_path + vectorstore + "docs.index") + # save to vectors directory + # check if the directory exists + if not os.path.exists(vectorstore): + os.makedirs(vectorstore) + + with open(vectorstore + "docs.index", "wb") as f: + f.write(r.content) + # download the store + r = requests.get(base_path + vectorstore + "faiss_store.pkl") + with open(vectorstore + "faiss_store.pkl", "wb") as f: + f.write(r.content) + + return {"status": 'loaded'} # handling CORS @app.after_request diff --git a/application/static/dist/css/output.css b/application/static/dist/css/output.css index 9d1609a7..1b5360e1 100644 --- a/application/static/dist/css/output.css +++ b/application/static/dist/css/output.css @@ -680,6 +680,11 @@ video { background-color: rgb(59 130 246 / var(--tw-bg-opacity)); } +.bg-gray-50 { + --tw-bg-opacity: 1; + background-color: rgb(249 250 251 / var(--tw-bg-opacity)); +} + .bg-gray-900 { --tw-bg-opacity: 1; background-color: rgb(17 24 39 / var(--tw-bg-opacity)); @@ -695,11 +700,6 @@ video { background-color: rgb(229 231 235 / var(--tw-bg-opacity)); } -.bg-gray-50 { - --tw-bg-opacity: 1; - background-color: rgb(249 250 251 / var(--tw-bg-opacity)); -} - .p-2 { padding: 0.5rem; } @@ -867,43 +867,6 @@ video { --tw-ring-color: rgb(59 130 246 / var(--tw-ring-opacity)); } -@media (prefers-color-scheme: dark) { - .dark\:border-gray-600 { - --tw-border-opacity: 1; - border-color: rgb(75 85 99 / var(--tw-border-opacity)); - } - - .dark\:bg-gray-700 { - --tw-bg-opacity: 1; - background-color: rgb(55 65 81 / var(--tw-bg-opacity)); - } - - .dark\:text-white { - --tw-text-opacity: 1; - color: rgb(255 255 255 / var(--tw-text-opacity)); - } - - .dark\:placeholder-gray-400::-moz-placeholder { - --tw-placeholder-opacity: 1; - color: rgb(156 163 175 / var(--tw-placeholder-opacity)); - } - - .dark\:placeholder-gray-400::placeholder { - --tw-placeholder-opacity: 1; - color: rgb(156 163 175 / var(--tw-placeholder-opacity)); - } - - .dark\:focus\:border-blue-500:focus { - --tw-border-opacity: 1; - border-color: rgb(59 130 246 / var(--tw-border-opacity)); - } - - .dark\:focus\:ring-blue-500:focus { - --tw-ring-opacity: 1; - --tw-ring-color: rgb(59 130 246 / var(--tw-ring-opacity)); - } -} - @media (min-width: 640px) { .sm\:my-8 { margin-top: 2rem; diff --git a/application/static/favicon/android-chrome-192x192.png b/application/static/favicon/android-chrome-192x192.png new file mode 100644 index 00000000..117c9009 Binary files /dev/null and b/application/static/favicon/android-chrome-192x192.png differ diff --git a/application/static/favicon/android-chrome-512x512.png b/application/static/favicon/android-chrome-512x512.png new file mode 100644 index 00000000..05b03a90 Binary files /dev/null and b/application/static/favicon/android-chrome-512x512.png differ diff --git a/application/static/favicon/apple-touch-icon.png b/application/static/favicon/apple-touch-icon.png new file mode 100644 index 00000000..0179ed0a Binary files /dev/null and b/application/static/favicon/apple-touch-icon.png differ diff --git a/application/static/favicon/favicon-16x16.png b/application/static/favicon/favicon-16x16.png new file mode 100644 index 00000000..0042d819 Binary files /dev/null and b/application/static/favicon/favicon-16x16.png differ diff --git a/application/static/favicon/favicon-32x32.png b/application/static/favicon/favicon-32x32.png new file mode 100644 index 00000000..3818e500 Binary files /dev/null and b/application/static/favicon/favicon-32x32.png differ diff --git a/application/static/favicon/favicon.ico b/application/static/favicon/favicon.ico new file mode 100644 index 00000000..1c5014d2 Binary files /dev/null and b/application/static/favicon/favicon.ico differ diff --git a/application/static/favicon/site.webmanifest b/application/static/favicon/site.webmanifest new file mode 100644 index 00000000..45dc8a20 --- /dev/null +++ b/application/static/favicon/site.webmanifest @@ -0,0 +1 @@ +{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"} \ No newline at end of file diff --git a/application/static/src/authapi.js b/application/static/src/authapi.js index d13d8aa0..ab634ef8 100644 --- a/application/static/src/authapi.js +++ b/application/static/src/authapi.js @@ -1,15 +1,19 @@ function resetApiKey() { - document.getElementById('modal').classList.toggle('hidden') + const modal = document.getElementById("modal"); + modal.classList.toggle("hidden"); } +const apiKeyForm = document.getElementById("api-key-form"); +if (apiKeyForm) { + apiKeyForm.addEventListener("submit", function(event) { + event.preventDefault(); -var el2 = document.getElementById('api-key-form'); -if (el2) { - el2.addEventListener("submit", function (event) { - event.preventDefault() - var apiKey = document.getElementById("api-key-input").value; - document.getElementById('modal').classList.toggle('hidden') - localStorage.setItem('apiKey', apiKey) - document.getElementById('api-key-input').value = '' - }); + const apiKeyInput = document.getElementById("api-key-input"); + const apiKey = apiKeyInput.value; + + localStorage.setItem("apiKey", apiKey); + + apiKeyInput.value = ""; + modal.classList.toggle("hidden"); + }); } diff --git a/application/static/src/chat.js b/application/static/src/chat.js index 862aba3e..832b8e27 100644 --- a/application/static/src/chat.js +++ b/application/static/src/chat.js @@ -13,6 +13,9 @@ if (el) { document.getElementById("message-input").value = ""; document.getElementById("button-submit").innerHTML = ' Thinking...'; document.getElementById("button-submit").disabled = true; + if (localStorage.getItem('activeDocs') == null) { + localStorage.setItem('activeDocs', 'default') + } fetch('/api/answer', { method: 'POST', @@ -22,7 +25,7 @@ if (el) { body: JSON.stringify({question: message, api_key: localStorage.getItem('apiKey'), - active_docs: localStorage.getItem('activeDocs'),}), + active_docs: localStorage.getItem('activeDocs')}), }) .then(response => response.json()) .then(data => { diff --git a/application/static/src/choiceChange.js b/application/static/src/choiceChange.js index 24887723..430ccc71 100644 --- a/application/static/src/choiceChange.js +++ b/application/static/src/choiceChange.js @@ -1,3 +1,15 @@ - document.getElementById("select-docs").addEventListener("change", function() { - localStorage.setItem('activeDocs', this.value) - }); \ No newline at end of file +document.getElementById("select-docs").addEventListener("change", function() { +localStorage.setItem('activeDocs', this.value) + fetch('/api/docs_check', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({docs: this.value}), + }).then(response => response.json()).then( + data => { + console.log('Success:', data); + } + ) +}); + diff --git a/application/templates/index.html b/application/templates/index.html index 50becfca..f916f148 100644 --- a/application/templates/index.html +++ b/application/templates/index.html @@ -3,6 +3,11 @@
The source code is available on Github
Currently It uses python pandas documentation, so it will respond to information relevant to pandas. If you want to train it on different documentation - please follow this guide
If you want to launch it on your own server - follow this guide