Merge pull request #1419 from fadingNA/pagination

Pagination

.env-template

@@ -1,4 +1,5 @@
 API_KEY=<LLM api key (for example, open ai key)>
+LLM_NAME=docsgpt
 VITE_API_STREAMING=true
 
 #For Azure (you can delete it if you don't use Azure)

.github/dependabot.yml

@@ -0,0 +1,19 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/application" # Location of package manifests
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "npm" # See documentation for possible values
+    directory: "/frontend" # Location of package manifests
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/holopin.yml

@@ -1,5 +1,11 @@
-organization: arc53
-defaultSticker: cln9dm7qz164460gk5ksrgr034
+organization: docsgpt
+defaultSticker: cm1ulwkkl180570cl82rtzympu
 stickers:
-  - id: cln9dm7qz164460gk5ksrgr034
-    alias: hacktober
+  - id: cm1ulwkkl180570cl82rtzympu
+    alias: contributor2024
+  - id: cm1ureg8o130450cl8c1po6mil
+    alias: api
+  - id: cm1urhmag148240cl8yvqxkthx
+    alias: lpc
+  - id: cm1urlcpq622090cl2tvu4w71y
+    alias: lexeu

.github/labeler.yml

@@ -1,23 +1,31 @@
 repo:
-  - '*'
+- changed-files:
+  - any-glob-to-any-file: '*'
 
 github:
-  - .github/**/*
+- changed-files:
+  - any-glob-to-any-file: '.github/**/*'
 
 application:
-  - application/**/*
+- changed-files:
+  - any-glob-to-any-file: 'application/**/*'
 
 docs:
-  - docs/**/*
+- changed-files:
+  - any-glob-to-any-file: 'docs/**/*'
 
 extensions:
-  - extensions/**/*
+- changed-files:
+  - any-glob-to-any-file: 'extensions/**/*'
 
 frontend:
-  - frontend/**/*
+- changed-files:
+  - any-glob-to-any-file: 'frontend/**/*'
 
 scripts:
-  - scripts/**/*
+- changed-files:
+  - any-glob-to-any-file: 'scripts/**/*'
 
 tests:
-  - tests/**/*
+- changed-files:
+  - any-glob-to-any-file: 'tests/**/*'

.github/workflows/ci.yml

@@ -1,10 +1,8 @@
 name: Build and push DocsGPT Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
@@ -13,37 +11,37 @@ jobs:
     permissions:
       contents: read
       packages: write
-
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Runs a single command using the runners shell
       - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           file: './application/Dockerfile'
           platforms: linux/amd64
           context: ./application
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+          cache-to: type=inline

.github/workflows/cife.yml

@@ -1,35 +1,33 @@
 name: Build and push DocsGPT-FE Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
+    if: github.repository == 'arc53/DocsGPT'
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
-
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -37,12 +35,14 @@ jobs:
 
       # Runs a single command using the runners shell
       - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           file: './frontend/Dockerfile'
-          platforms: linux/amd64
+          platforms: linux/amd64, linux/arm64
           context: ./frontend
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+          cache-to: type=inline

.github/workflows/docker-develop-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v6
+        with:
+          file: './application/Dockerfile'
+          platforms: linux/amd64
+          context: ./application
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+          cache-to: type=inline

.github/workflows/docker-develop-fe-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT FE Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v6
+        with:
+          file: './frontend/Dockerfile'
+          platforms: linux/amd64
+          context: ./frontend
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+          cache-to: type=inline

.github/workflows/labeler.yml

@@ -4,12 +4,13 @@ on:
   - pull_request_target
 jobs:
   triage:
+    if: github.repository == 'arc53/DocsGPT'
     permissions:
       contents: read
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/labeler@v4
+      - uses: actions/labeler@v5
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           sync-labels: true

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Lint with Ruff
         uses: chartboost/ruff-action@v1

.github/workflows/pytest.yml

@@ -6,11 +6,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.11"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -21,10 +21,10 @@ jobs:
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
       - name: Test with pytest and generate coverage report
         run: |
-          python -m pytest --cov=application --cov=scripts --cov=extensions --cov-report=xml
+          python -m pytest --cov=application --cov-report=xml
       - name: Upload coverage reports to Codecov
         if: github.event_name == 'pull_request' && matrix.python-version == '3.11'
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/sync_fork.yaml

@@ -17,7 +17,7 @@ jobs:
     steps:
       # Step 1: run a standard checkout action
       - name: Checkout target repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Step 2: run the sync action
       - name: Sync upstream changes

.gitignore

@@ -75,6 +75,7 @@ target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
+**/*.ipynb
 
 # IPython
 profile_default/
@@ -171,4 +172,5 @@ application/vectors/
 
 node_modules/
 .vscode/settings.json
-models/
+/models/
+model/

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Docker Debug Frontend",
+      "request": "launch",
+      "type": "chrome",
+      "preLaunchTask": "docker-compose: debug:frontend",
+      "url": "http://127.0.0.1:5173",
+      "webRoot": "${workspaceFolder}/frontend",
+      "skipFiles": [
+        "<node_internals>/**"
+      ]
+    }
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,21 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "docker-compose",
+      "label": "docker-compose: debug:frontend",
+      "dockerCompose": {
+        "up": {
+          "detached": true,
+          "services": [
+            "frontend"
+          ],
+          "build": true
+        },
+        "files": [
+          "${workspaceFolder}/docker-compose.yaml"
+        ]
+      }
+    }
+  ]
+}

CONTRIBUTING.md

@@ -6,7 +6,7 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 
 📣 **Discussions** - Engage in conversations, start new topics, or help answer questions.
 
-🐞 **Issues** - This is where we keep track of tasks. It could be bugs,fixes or suggestions for new features.
+🐞 **Issues** - This is where we keep track of tasks. It could be bugs, fixes or suggestions for new features.
 
 🛠️ **Pull requests** - Suggest changes to our repository, either by working on existing issues or adding new features.
 
@@ -21,8 +21,9 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 - If you're interested in contributing code, here are some important things to know:
 
 - We have a frontend built on React (Vite) and a backend in Python.
-=======
-Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version that you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
+
+  
+Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
 
 ### 👨‍💻 If you're interested in contributing code, here are some important things to know:
 
@@ -43,7 +44,7 @@ Please try to follow the guidelines.
 
 ### 🖥 If you are looking to contribute to Backend (🐍 Python):
 
-- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) or [`/scripts`](https://github.com/arc53/DocsGPT/tree/main/scripts) (please disregard old [`ingest_rst.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst.py) [`ingest_rst_sphinx.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst_sphinx.py) files; they will be deprecated soon).
+- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) or [`/scripts`](https://github.com/arc53/DocsGPT/tree/main/scripts) (please disregard old [`ingest_rst.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst.py) [`ingest_rst_sphinx.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst_sphinx.py) files; these will be deprecated soon).
 - All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
 - Before submitting your Pull Request, ensure it can be queried after ingesting some test data.
   
@@ -125,4 +126,4 @@ Thank you for considering contributing to DocsGPT! 🙏
 
 ## Questions/collaboration
 Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU). We're very friendly and welcoming to new contributors, so don't hesitate to reach out.
-# Thank you so much for considering to contribute DocsGPT!🙏
+# Thank you so much for considering to contributing DocsGPT!🙏

HACKTOBERFEST.md

@@ -0,0 +1,41 @@
+# **🎉 Join the Hacktoberfest with DocsGPT and win a Free T-shirt and other prizes! 🎉**
+
+Welcome, contributors! We're excited to announce that DocsGPT is participating in Hacktoberfest. Get involved by submitting meaningful pull requests.
+
+All contributors with accepted PRs will receive a cool Holopin! 🤩 (Watch out for a reply in your PR to collect it).
+
+### 🏆 Top 50 contributors will receive a special T-shirt
+
+### 🏆 [LLM Document analysis by LexEU competition](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md): 
+A separate competition is available for those who submit new retrieval / workflow method that will analyze a Document using EU laws.
+With 200$, 100$, 50$ prize for 1st, 2nd and 3rd place respectively.
+You can find more information [here](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md)
+
+## 📜 Here's How to Contribute:
+```text
+🛠️ Code: This is the golden ticket! Make meaningful contributions through PRs.
+
+🧩 API extension: Build an app utilising DocsGPT API. We prefer submissions that showcase original ideas and turn the API into an AI agent.
+They can be a completely separate repos. 
+For example: 
+https://github.com/arc53/tg-bot-docsgpt-extenstion or 
+https://github.com/arc53/DocsGPT-cli
+
+Non-Code Contributions:
+
+📚 Wiki: Improve our documentation, create a guide or change existing documentation.
+
+🖥️ Design: Improve the UI/UX or design a new feature.
+
+📝 Blogging or Content Creation: Write articles or create videos to showcase DocsGPT or highlight your contributions!
+```
+
+### 📝 Guidelines for Pull Requests:
+- Familiarize yourself with the current contributions and our [Roadmap](https://github.com/orgs/arc53/projects/2).
+- Before contributing we highly advise that you check existing [issues](https://github.com/arc53/DocsGPT/issues) or [create](https://github.com/arc53/DocsGPT/issues/new/choose) an issue and wait to get assigned.
+- Once you are finished with your contribution, please fill in this [form](https://airtable.com/appikMaJwdHhC1SDP/pagoblCJ9W29wf6Hf/form).
+- Refer to the [Documentation](https://docs.docsgpt.cloud/).
+- Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU) server. We're here to help newcomers, so don't hesitate to jump in! Join us [here](https://discord.gg/n5BX8dh8rU).
+  
+Thank you very much for considering contributing to DocsGPT during Hacktoberfest! 🙏 Your contributions (not just simple typos) could earn you a stylish new t-shirt and other prizes as a token of our appreciation. 🎁 Join us, and let's code together! 🚀
+

README.md

@@ -7,9 +7,9 @@
 </p>
 
 <p align="left">
-  <strong><a href="https://docsgpt.arc53.com/">DocsGPT</a></strong> is a cutting-edge open-source solution that streamlines the process of finding information in the project documentation. With its integration of the powerful <strong>GPT</strong> models, developers can easily ask questions about a project and receive accurate answers.
+  <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> is a cutting-edge open-source solution that streamlines the process of finding information in the project documentation. With its integration of the powerful <strong>GPT</strong> models, developers can easily ask questions about a project and receive accurate answers.
   
-Say goodbye to time-consuming manual searches, and let <strong><a href="https://docsgpt.arc53.com/">DocsGPT</a></strong> help you quickly find the information you need. Try it out and see how it revolutionizes your project documentation experience. Contribute to its development and be a part of the future of AI-powered assistance.
+Say goodbye to time-consuming manual searches, and let <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> help you quickly find the information you need. Try it out and see how it revolutionizes your project documentation experience. Contribute to its development and be a part of the future of AI-powered assistance.
 </p>
 
 <div align="center">
@@ -18,7 +18,7 @@ Say goodbye to time-consuming manual searches, and let <strong><a href="https://
   <a href="https://github.com/arc53/DocsGPT">![link to main GitHub showing Forks number](https://img.shields.io/github/forks/arc53/docsgpt?style=social)</a>
   <a href="https://github.com/arc53/DocsGPT/blob/main/LICENSE">![link to license file](https://img.shields.io/github/license/arc53/docsgpt)</a>
   <a href="https://discord.gg/n5BX8dh8rU">![link to discord](https://img.shields.io/discord/1070046503302877216)</a>
-  <a href="https://twitter.com/ATushynski">![X (formerly Twitter) URL](https://img.shields.io/twitter/follow/ATushynski)</a>
+  <a href="https://twitter.com/docsgptai">![X (formerly Twitter) URL](https://img.shields.io/twitter/follow/docsgptai)</a>
 
  
 </div>
@@ -27,10 +27,14 @@ Say goodbye to time-consuming manual searches, and let <strong><a href="https://
 
 We're eager to provide personalized assistance when deploying your DocsGPT to a live environment.
 
-- [Book Demo :wave:](https://airtable.com/appdeaL0F1qV8Bl2C/shrrJF1Ll7btCJRbP)
-- [Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
+<a href ="https://cal.com/arc53/docsgpt-demo-b2b">
+    <img alt="Let's chat" src="https://cal.com/book-with-cal-dark.svg" />
+</a>
 
-![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
+[Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
+
+
+<img src="https://github.com/user-attachments/assets/9a1f21de-7a15-4e42-9424-70d22ba5a913" alt="video-example-of-docs-gpt" width="1000" height="500">
 
 ## Roadmap
 
@@ -40,29 +44,29 @@ You can find our roadmap [here](https://github.com/orgs/arc53/projects/2). Pleas
 
 | Name                                                                  | Base Model  | Requirements (or similar) |
 | --------------------------------------------------------------------- | ----------- | ------------------------- |
-| [Docsgpt-7b-falcon](https://huggingface.co/Arc53/docsgpt-7b-falcon)   | Falcon-7b   | 1xA10G gpu                |
+| [Docsgpt-7b-mistral](https://huggingface.co/Arc53/docsgpt-7b-mistral)   | Mistral-7b   | 1xA10G gpu                |
 | [Docsgpt-14b](https://huggingface.co/Arc53/docsgpt-14b)               | llama-2-14b | 2xA10 gpu's               |
 | [Docsgpt-40b-falcon](https://huggingface.co/Arc53/docsgpt-40b-falcon) | falcon-40b  | 8xA10G gpu's              |
 
 If you don't have enough resources to run it, you can use bitsnbytes to quantize.
 
-## Features
+## End to End AI Framework for Information Retrieval
 
-![Main features of DocsGPT showcasing six main features](https://user-images.githubusercontent.com/17906039/220427472-2644cff4-7666-46a5-819f-fc4a521f63c7.png)
+![Architecture chart](https://github.com/user-attachments/assets/fc6a7841-ddfc-45e6-b5a0-d05fe648cbe2)
 
 ## Useful Links
 
-- :mag: :fire: [Live preview](https://docsgpt.arc53.com/)
+- :mag: :fire: [Cloud Version](https://app.docsgpt.cloud/)
 
 - :speech_balloon: :tada: [Join our Discord](https://discord.gg/n5BX8dh8rU)
 
-- :books: :sunglasses: [Guides](https://docs.docsgpt.co.uk/)
+- :books: :sunglasses: [Guides](https://docs.docsgpt.cloud/)
 
 - :couple: [Interested in contributing?](https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md)
 
-- :file_folder: :rocket: [How to use any other documentation](https://docs.docsgpt.co.uk/Guides/How-to-train-on-other-documentation)
+- :file_folder: :rocket: [How to use any other documentation](https://docs.docsgpt.cloud/Guides/How-to-train-on-other-documentation)
 
-- :house: :closed_lock_with_key: [How to host it locally (so all data will stay on-premises)](https://docs.docsgpt.co.uk/Guides/How-to-use-different-LLM)
+- :house: :closed_lock_with_key: [How to host it locally (so all data will stay on-premises)](https://docs.docsgpt.cloud/Guides/How-to-use-different-LLM)
 
 ## Project Structure
 
@@ -83,17 +87,18 @@ On Mac OS or Linux, write:
 
 `./setup.sh`
 
-It will install all the dependencies and allow you to download the local model or use OpenAI.
+It will install all the dependencies and allow you to download the local model, use OpenAI or use our LLM API.
 
-Otherwise, refer to this Guide:
+Otherwise, refer to this Guide for Windows:
 
 1. Download and open this repository with `git clone https://github.com/arc53/DocsGPT.git`
-2. Create a `.env` file in your root directory and set the env variable `API_KEY` with your [OpenAI API key](https://platform.openai.com/account/api-keys) and `VITE_API_STREAMING` to true or false, depending on whether you want streaming answers or not.
+2. Create a `.env` file in your root directory and set the env variables and `VITE_API_STREAMING` to true or false, depending on whether you want streaming answers or not.
    It should look like this inside:
 
    ```
-   API_KEY=Yourkey
+   LLM_NAME=[docsgpt or openai or others] 
    VITE_API_STREAMING=true
+   API_KEY=[if LLM_NAME is openai]
    ```
 
    See optional environment variables in the [/.env-template](https://github.com/arc53/DocsGPT/blob/main/.env-template) and [/application/.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) files.
@@ -122,8 +127,8 @@ docker compose -f docker-compose-dev.yaml up -d
 > [!Note]
 > Make sure you have Python 3.10 or 3.11 installed.
 
-1. Export required environment variables or prepare a `.env` file in the `/application` folder:
-   - Copy [.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) and create `.env` with your OpenAI API token for the `API_KEY` and `EMBEDDINGS_KEY` fields.
+1. Export required environment variables or prepare a `.env` file in the project folder:
+   - Copy [.env-template](https://github.com/arc53/DocsGPT/blob/main/application/.env-template) and create `.env`.
 
 (check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.)
 
@@ -144,14 +149,23 @@ python -m venv venv
  venv/Scripts/activate
 ```
 
-3. Change to the `application/` subdir by the command `cd application/` and install dependencies for the backend:
+3. Download embedding model and save it in the `model/` folder:
+You can use the script below, or download it manually from [here](https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip), unzip it and save it in the `model/` folder.
+
+```commandline
+wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip
+unzip mpnet-base-v2.zip -d model
+rm mpnet-base-v2.zip
+```
+
+4. Install dependencies for the backend:
 
 ```commandline
 pip install -r application/requirements.txt
 ```
 
-4. Run the app using `flask --app application/app.py run --host=0.0.0.0 --port=7091`.
-5. Start worker with `celery -A application.app.celery worker -l INFO`.
+5. Run the app using `flask --app application/app.py run --host=0.0.0.0 --port=7091`.
+6. Start worker with `celery -A application.app.celery worker -l INFO`.
 
 ### Start Frontend
 
@@ -187,4 +201,9 @@ We as members, contributors, and leaders, pledge to make participation in our co
 
 The source code license is [MIT](https://opensource.org/license/mit/), as described in the [LICENSE](LICENSE) file.
 
-Built with [:bird: :link: LangChain](https://github.com/hwchase17/langchain)
+<p>This project is supported by:</p>
+<p>
+  <a href="https://www.digitalocean.com/?utm_medium=opensource&utm_source=DocsGPT">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px">
+  </a>
+</p>

SECURITY.md

@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+Supported Versions:
+
+Currently, we support security patches by committing changes and bumping the version published on Github.
+
+## Reporting a Vulnerability
+
+Found a vulnerability? Please email us:
+
+security@arc53.com
+

application/Dockerfile

@@ -1,23 +1,88 @@
-FROM python:3.10-slim-bullseye as builder
+# Builder Stage
+FROM ubuntu:24.04 as builder
 
-# Tiktoken requires Rust toolchain, so build it in a separate stage
-RUN apt-get update && apt-get install -y gcc curl
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && apt-get install --reinstall libc6-dev -y
-ENV PATH="/root/.cargo/bin:${PATH}"
-RUN pip install --upgrade pip && pip install tiktoken==0.3.3
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+# Install necessary packages and Python
+    apt-get update && \
+    apt-get install -y --no-install-recommends gcc wget unzip libc6-dev python3.11 python3.11-distutils python3.11-venv && \
+    rm -rf /var/lib/apt/lists/* 
+
+# Verify Python installation and setup symlink
+RUN if [ -f /usr/bin/python3.11 ]; then \
+        ln -s /usr/bin/python3.11 /usr/bin/python; \
+    else \
+        echo "Python 3.11 not found"; exit 1; \
+    fi
+
+# Download and unzip the model
+RUN wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip && \
+    unzip mpnet-base-v2.zip -d model && \
+    rm mpnet-base-v2.zip
+
+# Install Rust
+RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y
+
+# Clean up to reduce container size
+RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements.txt
 COPY requirements.txt .
-RUN pip install -r requirements.txt
 
-FROM python:3.10-slim-bullseye
+# Setup Python virtual environment
+RUN python3.11 -m venv /venv
 
-# Copy pre-built packages and binaries from builder stage
-COPY --from=builder /usr/local/ /usr/local/
+# Activate virtual environment and install Python packages
+ENV PATH="/venv/bin:$PATH"
 
+# Install Python packages
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir tiktoken && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Final Stage
+FROM ubuntu:24.04 as final
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+# Install Python
+    apt-get update && apt-get install -y --no-install-recommends python3.11 && \
+    ln -s /usr/bin/python3.11 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set working directory
 WORKDIR /app
-COPY . /app/application
-ENV FLASK_APP=app.py
-ENV FLASK_DEBUG=true
 
+# Create a non-root user: `appuser` (Feel free to choose a name)
+RUN groupadd -r appuser && \
+    useradd -r -g appuser -d /app -s /sbin/nologin -c "Docker image user" appuser
+
+# Copy the virtual environment and model from the builder stage
+COPY --from=builder /venv /venv
+COPY --from=builder /model /app/model
+
+# Copy your application code
+COPY . /app/application
+
+# Change the ownership of the /app directory to the appuser
+	
+RUN mkdir -p /app/application/inputs/local
+RUN chown -R appuser:appuser /app
+
+# Set environment variables
+ENV FLASK_APP=app.py \
+    FLASK_DEBUG=true \
+    PATH="/venv/bin:$PATH"
+
+# Expose the port the app runs on
 EXPOSE 7091
 
-CMD ["gunicorn", "-w", "2", "--timeout", "120", "--bind", "0.0.0.0:7091", "application.wsgi:app"]
+# Switch to non-root user
+USER appuser
+
+# Start Gunicorn
+CMD ["gunicorn", "-w", "2", "--timeout", "120", "--bind", "0.0.0.0:7091", "application.wsgi:app"]

application/api/answer/routes.py

@@ -1,42 +1,55 @@
 import asyncio
-import os
-from flask import Blueprint, request, Response
-import json
 import datetime
+import json
 import logging
+import os
+import sys
 import traceback
 
-from pymongo import MongoClient
+from bson.dbref import DBRef
 from bson.objectid import ObjectId
-from transformers import GPT2TokenizerFast
-
+from flask import Blueprint, current_app, make_response, request, Response
+from flask_restx import fields, Namespace, Resource
 
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
-from application.vectorstore.vector_creator import VectorCreator
-from application.llm.llm_creator import LLMCreator
 from application.error import bad_request
-
-
+from application.extensions import api
+from application.llm.llm_creator import LLMCreator
+from application.retriever.retriever_creator import RetrieverCreator
+from application.utils import check_required_fields
 
 logger = logging.getLogger(__name__)
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 prompts_collection = db["prompts"]
-answer = Blueprint('answer', __name__)
+api_key_collection = db["api_keys"]
+user_logs_collection = db["user_logs"]
 
-if settings.LLM_NAME == "gpt4":
-    gpt_model = 'gpt-4'
+answer = Blueprint("answer", __name__)
+answer_ns = Namespace("answer", description="Answer related operations", path="/")
+api.add_namespace(answer_ns)
+
+gpt_model = ""
+# to have some kind of default behaviour
+if settings.LLM_NAME == "openai":
+    gpt_model = "gpt-3.5-turbo"
 elif settings.LLM_NAME == "anthropic":
-    gpt_model = 'claude-2'
-else:
-    gpt_model = 'gpt-3.5-turbo'
+    gpt_model = "claude-2"
+elif settings.LLM_NAME == "groq":
+    gpt_model = "llama3-8b-8192"
+
+if settings.MODEL_NAME:  # in case there is particular model name configured
+    gpt_model = settings.MODEL_NAME
 
 # load the prompts
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
 with open(os.path.join(current_dir, "prompts", "chat_combine_default.txt"), "r") as f:
     chat_combine_template = f.read()
 
@@ -47,7 +60,7 @@ with open(os.path.join(current_dir, "prompts", "chat_combine_creative.txt"), "r"
     chat_combine_creative = f.read()
 
 with open(os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r") as f:
-    chat_combine_strict = f.read()    
+    chat_combine_strict = f.read()
 
 api_key_set = settings.API_KEY is not None
 embeddings_key_set = settings.EMBEDDINGS_KEY is not None
@@ -58,11 +71,6 @@ async def async_generate(chain, question, chat_history):
     return result
 
 
-def count_tokens(string):
-    tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
-    return len(tokenizer(string)['input_ids'])
-
-
 def run_async_chain(chain, question, chat_history):
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
@@ -75,271 +83,536 @@ def run_async_chain(chain, question, chat_history):
     return result
 
 
-def get_vectorstore(data):
-    if "active_docs" in data:
-        if data["active_docs"].split("/")[0] == "default":
-                vectorstore = ""
-        elif data["active_docs"].split("/")[0] == "local":
-            vectorstore = "indexes/" + data["active_docs"]
-        else:
-            vectorstore = "vectors/" + data["active_docs"]
-        if data["active_docs"] == "default":
-            vectorstore = ""
+def get_data_from_api_key(api_key):
+    data = api_key_collection.find_one({"key": api_key})
+    # # Raise custom exception if the API key is not found
+    if data is None:
+        raise Exception("Invalid API Key, please generate new key", 401)
+
+    if "retriever" not in data:
+        data["retriever"] = None
+
+    if "source" in data and isinstance(data["source"], DBRef):
+        source_doc = db.dereference(data["source"])
+        data["source"] = str(source_doc["_id"])
+        if "retriever" in source_doc:
+            data["retriever"] = source_doc["retriever"]
     else:
-        vectorstore = ""
-    vectorstore = os.path.join("application", vectorstore)
-    return vectorstore
+        data["source"] = {}
+    return data
+
+
+def get_retriever(source_id: str):
+    doc = sources_collection.find_one({"_id": ObjectId(source_id)})
+    if doc is None:
+        raise Exception("Source document does not exist", 404)
+    retriever_name = None if "retriever" not in doc else doc["retriever"]
+    return retriever_name
 
 
 def is_azure_configured():
-    return settings.OPENAI_API_BASE and settings.OPENAI_API_VERSION and settings.AZURE_DEPLOYMENT_NAME
+    return (
+        settings.OPENAI_API_BASE
+        and settings.OPENAI_API_VERSION
+        and settings.AZURE_DEPLOYMENT_NAME
+    )
 
 
-def complete_stream(question, docsearch, chat_history, api_key, prompt_id, conversation_id):
-    llm = LLMCreator.create_llm(settings.LLM_NAME, api_key=api_key)
-
-    if prompt_id == 'default':
-        prompt = chat_combine_template
-    elif prompt_id == 'creative':
-        prompt = chat_combine_creative
-    elif prompt_id == 'strict':
-        prompt = chat_combine_strict
-    else:
-        prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})["content"]
-
-    docs = docsearch.search(question, k=2)
-    if settings.LLM_NAME == "llama.cpp":
-        docs = [docs[0]]
-    # join all page_content together with a newline
-    docs_together = "\n".join([doc.page_content for doc in docs])
-    p_chat_combine = prompt.replace("{summaries}", docs_together)
-    messages_combine = [{"role": "system", "content": p_chat_combine}]
-    source_log_docs = []
-    for doc in docs:
-        if doc.metadata:
-            data = json.dumps({"type": "source", "doc": doc.page_content, "metadata": doc.metadata})
-            source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content})
-        else:
-            data = json.dumps({"type": "source", "doc": doc.page_content})
-            source_log_docs.append({"title": doc.page_content, "text": doc.page_content})
-        yield f"data:{data}\n\n"
-
-    if len(chat_history) > 1:
-        tokens_current_history = 0
-        # count tokens in history
-        chat_history.reverse()
-        for i in chat_history:
-            if "prompt" in i and "response" in i:
-                tokens_batch = count_tokens(i["prompt"]) + count_tokens(i["response"])
-                if tokens_current_history + tokens_batch < settings.TOKENS_MAX_HISTORY:
-                    tokens_current_history += tokens_batch
-                    messages_combine.append({"role": "user", "content": i["prompt"]})
-                    messages_combine.append({"role": "system", "content": i["response"]})
-    messages_combine.append({"role": "user", "content": question})
-
-    response_full = ""
-    completion = llm.gen_stream(model=gpt_model, engine=settings.AZURE_DEPLOYMENT_NAME,
-                                messages=messages_combine)
-    for line in completion:
-        data = json.dumps({"answer": str(line)})
-        response_full += str(line)
-        yield f"data: {data}\n\n"
-
-    # save conversation to database
-    if conversation_id is not None:
+def save_conversation(conversation_id, question, response, source_log_docs, llm):
+    if conversation_id is not None and conversation_id != "None":
         conversations_collection.update_one(
             {"_id": ObjectId(conversation_id)},
-            {"$push": {"queries": {"prompt": question, "response": response_full, "sources": source_log_docs}}},
+            {
+                "$push": {
+                    "queries": {
+                        "prompt": question,
+                        "response": response,
+                        "sources": source_log_docs,
+                    }
+                }
+            },
         )
 
     else:
         # create new conversation
         # generate summary
-        messages_summary = [{"role": "assistant", "content": "Summarise following conversation in no more than 3 "
-                                                             "words, respond ONLY with the summary, use the same "
-                                                             "language as the system \n\nUser: " + question + "\n\n" +
-                                                             "AI: " +
-                                                             response_full},
-                            {"role": "user", "content": "Summarise following conversation in no more than 3 words, "
-                                                        "respond ONLY with the summary, use the same language as the "
-                                                        "system"}]
+        messages_summary = [
+            {
+                "role": "assistant",
+                "content": "Summarise following conversation in no more than 3 "
+                "words, respond ONLY with the summary, use the same "
+                "language as the system \n\nUser: "
+                + question
+                + "\n\n"
+                + "AI: "
+                + response,
+            },
+            {
+                "role": "user",
+                "content": "Summarise following conversation in no more than 3 words, "
+                "respond ONLY with the summary, use the same language as the "
+                "system",
+            },
+        ]
 
-        completion = llm.gen(model=gpt_model, engine=settings.AZURE_DEPLOYMENT_NAME,
-                             messages=messages_summary, max_tokens=30)
+        completion = llm.gen(model=gpt_model, messages=messages_summary, max_tokens=30)
         conversation_id = conversations_collection.insert_one(
-            {"user": "local",
-             "date": datetime.datetime.utcnow(),
-             "name": completion,
-             "queries": [{"prompt": question, "response": response_full, "sources": source_log_docs}]}
+            {
+                "user": "local",
+                "date": datetime.datetime.utcnow(),
+                "name": completion,
+                "queries": [
+                    {
+                        "prompt": question,
+                        "response": response,
+                        "sources": source_log_docs,
+                    }
+                ],
+            }
         ).inserted_id
-
-    # send data.type = "end" to indicate that the stream has ended as json
-    data = json.dumps({"type": "id", "id": str(conversation_id)})
-    yield f"data: {data}\n\n"
-    data = json.dumps({"type": "end"})
-    yield f"data: {data}\n\n"
+    return conversation_id
 
 
-@answer.route("/stream", methods=["POST"])
-def stream():
-    data = request.get_json()
-    # get parameter from url question
-    question = data["question"]
-    history = data["history"]
-    # history to json object from string
-    history = json.loads(history)
-    conversation_id = data["conversation_id"]
-    if 'prompt_id' in data:
-        prompt_id = data["prompt_id"]
-    else:
-        prompt_id = 'default'
-
-    # check if active_docs is set
-
-    if not api_key_set:
-        api_key = data["api_key"]
-    else:
-        api_key = settings.API_KEY
-    if not embeddings_key_set:
-        embeddings_key = data["embeddings_key"]
-    else:
-        embeddings_key = settings.EMBEDDINGS_KEY
-    if "active_docs" in data:
-        vectorstore = get_vectorstore({"active_docs": data["active_docs"]})
-    else:
-        vectorstore = ""
-    docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key)
-
-    return Response(
-        complete_stream(question, docsearch,
-                        chat_history=history, api_key=api_key,
-                        prompt_id=prompt_id,
-                        conversation_id=conversation_id), mimetype="text/event-stream"
-    )
-
-
-@answer.route("/api/answer", methods=["POST"])
-def api_answer():
-    data = request.get_json()
-    question = data["question"]
-    history = data["history"]
-    if "conversation_id" not in data:
-        conversation_id = None
-    else:
-        conversation_id = data["conversation_id"]
-    print("-" * 5)
-    if not api_key_set:
-        api_key = data["api_key"]
-    else:
-        api_key = settings.API_KEY
-    if not embeddings_key_set:
-        embeddings_key = data["embeddings_key"]
-    else:
-        embeddings_key = settings.EMBEDDINGS_KEY
-    if 'prompt_id' in data:
-        prompt_id = data["prompt_id"]
-    else:
-        prompt_id = 'default'
-
-    if prompt_id == 'default':
+def get_prompt(prompt_id):
+    if prompt_id == "default":
         prompt = chat_combine_template
-    elif prompt_id == 'creative':
+    elif prompt_id == "creative":
         prompt = chat_combine_creative
-    elif prompt_id == 'strict':
+    elif prompt_id == "strict":
         prompt = chat_combine_strict
     else:
         prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})["content"]
+    return prompt
+
+
+def complete_stream(
+    question, retriever, conversation_id, user_api_key, isNoneDoc=False
+):
 
-    # use try and except  to check for exception
     try:
-        # check if the vectorstore is set
-        vectorstore = get_vectorstore(data)
-        # loading the index and the store and the prompt template
-        # Note if you have used other embeddings than OpenAI, you need to change the embeddings
-        docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key)
-
-
-        llm = LLMCreator.create_llm(settings.LLM_NAME, api_key=api_key)
-
-
-
-        docs = docsearch.search(question, k=2)
-        # join all page_content together with a newline
-        docs_together = "\n".join([doc.page_content for doc in docs])
-        p_chat_combine = prompt.replace("{summaries}", docs_together)
-        messages_combine = [{"role": "system", "content": p_chat_combine}]
+        response_full = ""
         source_log_docs = []
-        for doc in docs:
-            if doc.metadata:
-                source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content})
-            else:
-                source_log_docs.append({"title": doc.page_content, "text": doc.page_content})
-        # join all page_content together with a newline
+        answer = retriever.gen()
+        sources = retriever.search()
+        for source in sources:
+            if "text" in source:
+                source["text"] = source["text"][:100].strip() + "..."
+        if len(sources) > 0:
+            data = json.dumps({"type": "source", "source": sources})
+            yield f"data: {data}\n\n"
+        for line in answer:
+            if "answer" in line:
+                response_full += str(line["answer"])
+                data = json.dumps(line)
+                yield f"data: {data}\n\n"
+            elif "source" in line:
+                source_log_docs.append(line["source"])
 
+        if isNoneDoc:
+            for doc in source_log_docs:
+                doc["source"] = "None"
 
-        if len(history) > 1:
-            tokens_current_history = 0
-            # count tokens in history
-            history.reverse()
-            for i in history:
-                if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(i["response"])
-                    if tokens_current_history + tokens_batch < settings.TOKENS_MAX_HISTORY:
-                        tokens_current_history += tokens_batch
-                        messages_combine.append({"role": "user", "content": i["prompt"]})
-                        messages_combine.append({"role": "system", "content": i["response"]})
-        messages_combine.append({"role": "user", "content": question})
-
-
-        completion = llm.gen(model=gpt_model, engine=settings.AZURE_DEPLOYMENT_NAME,
-                                    messages=messages_combine)
-
-
-        result = {"answer": completion, "sources": source_log_docs}
-        logger.debug(result)
-
-        # generate conversationId
-        if conversation_id is not None:
-            conversations_collection.update_one(
-                {"_id": ObjectId(conversation_id)},
-                {"$push": {"queries": {"prompt": question,
-                                       "response": result["answer"], "sources": result['sources']}}},
+        llm = LLMCreator.create_llm(
+            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
+        )
+        if user_api_key is None:
+            conversation_id = save_conversation(
+                conversation_id, question, response_full, source_log_docs, llm
             )
+            # send data.type = "end" to indicate that the stream has ended as json
+            data = json.dumps({"type": "id", "id": str(conversation_id)})
+            yield f"data: {data}\n\n"
 
-        else:
-            # create new conversation
-            # generate summary
-            messages_summary = [
-                {"role": "assistant", "content": "Summarise following conversation in no more than 3 words, "
-                    "respond ONLY with the summary, use the same language as the system \n\n"
-                    "User: " + question + "\n\n" + "AI: " + result["answer"]},
-                {"role": "user", "content": "Summarise following conversation in no more than 3 words, "
-                    "respond ONLY with the summary, use the same language as the system"}
-            ]
-
-            completion = llm.gen(
-                model=gpt_model,
-                engine=settings.AZURE_DEPLOYMENT_NAME,
-                messages=messages_summary,
-                max_tokens=30
-            )
-            conversation_id = conversations_collection.insert_one(
-                {"user": "local",
-                "date": datetime.datetime.utcnow(),
-                "name": completion,
-                "queries": [{"prompt": question, "response": result["answer"], "sources": source_log_docs}]}
-            ).inserted_id
-
-        result["conversation_id"] = str(conversation_id)
-
-        # mock result
-        # result = {
-        #     "answer": "The answer is 42",
-        #     "sources": ["https://en.wikipedia.org/wiki/42_(number)", "https://en.wikipedia.org/wiki/42_(number)"]
-        # }
-        return result
+        retriever_params = retriever.get_params()
+        user_logs_collection.insert_one(
+            {
+                "action": "stream_answer",
+                "level": "info",
+                "user": "local",
+                "api_key": user_api_key,
+                "question": question,
+                "response": response_full,
+                "sources": source_log_docs,
+                "retriever_params": retriever_params,
+                "timestamp": datetime.datetime.now(datetime.timezone.utc),
+            }
+        )
+        data = json.dumps({"type": "end"})
+        yield f"data: {data}\n\n"
     except Exception as e:
-        # print whole traceback
-        traceback.print_exc()
-        print(str(e))
-        return bad_request(500, str(e))
+        print("\033[91merr", str(e), file=sys.stderr)
+        data = json.dumps(
+            {
+                "type": "error",
+                "error": "Please try again later. We apologize for any inconvenience.",
+                "error_exception": str(e),
+            }
+        )
+        yield f"data: {data}\n\n"
+        return
+
+
+@answer_ns.route("/stream")
+class Stream(Resource):
+    stream_model = api.model(
+        "StreamModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
+            ),
+            "history": fields.List(
+                fields.String, required=False, description="Chat history"
+            ),
+            "conversation_id": fields.String(
+                required=False, description="Conversation ID"
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
+    )
+
+    @api.expect(stream_model)
+    @api.doc(description="Stream a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]
+
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            history = data.get("history", [])
+            history = json.loads(history)
+            conversation_id = data.get("conversation_id")
+            prompt_id = data.get("prompt_id", "default")
+            
+
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                prompt_id = data_key.get("prompt_id", "default")
+                source = {"active_docs": data_key.get("source")}
+                retriever_name = data_key.get("retriever", retriever_name)
+                user_api_key = data["api_key"]
+
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                retriever_name = get_retriever(data["active_docs"]) or retriever_name
+                user_api_key = None
+
+            else:
+                source = {}
+                user_api_key = None
+
+            current_app.logger.info(
+                f"/stream - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            prompt = get_prompt(prompt_id)
+            if "isNoneDoc" in data and data["isNoneDoc"] is True:
+                chunks = 0
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=history,
+                prompt=prompt,
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            return Response(
+                complete_stream(
+                    question=question,
+                    retriever=retriever,
+                    conversation_id=conversation_id,
+                    user_api_key=user_api_key,
+                    isNoneDoc=data.get("isNoneDoc"),
+                ),
+                mimetype="text/event-stream",
+            )
+
+        except ValueError:
+            message = "Malformed request body"
+            print("\033[91merr", str(message), file=sys.stderr)
+            return Response(
+                error_stream_generate(message),
+                status=400,
+                mimetype="text/event-stream",
+            )
+        except Exception as e:
+            current_app.logger.error(
+                f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            message = e.args[0]
+            status_code = 400
+            # Custom exceptions with two arguments, index 1 as status code
+            if len(e.args) >= 2:
+                status_code = e.args[1]
+            return Response(
+                error_stream_generate(message),
+                status=status_code,
+                mimetype="text/event-stream",
+            )
+
+
+def error_stream_generate(err_response):
+    data = json.dumps({"type": "error", "error": err_response})
+    yield f"data: {data}\n\n"
+
+
+@answer_ns.route("/api/answer")
+class Answer(Resource):
+    answer_model = api.model(
+        "AnswerModel",
+        {
+            "question": fields.String(
+                required=True, description="The question to answer"
+            ),
+            "history": fields.List(
+                fields.String, required=False, description="Conversation history"
+            ),
+            "conversation_id": fields.String(
+                required=False, description="Conversation ID"
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
+    )
+
+    @api.expect(answer_model)
+    @api.doc(description="Provide an answer based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]       
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            history = data.get("history", [])
+            conversation_id = data.get("conversation_id")
+            prompt_id = data.get("prompt_id", "default")
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                prompt_id = data_key.get("prompt_id", "default")
+                source = {"active_docs": data_key.get("source")}
+                retriever_name = data_key.get("retriever", retriever_name)
+                user_api_key = data["api_key"]
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                retriever_name = get_retriever(data["active_docs"]) or retriever_name
+                user_api_key = None
+            else:
+                source = {}
+                user_api_key = None
+
+            prompt = get_prompt(prompt_id)
+
+            current_app.logger.info(
+                f"/api/answer - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=history,
+                prompt=prompt,
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            source_log_docs = []
+            response_full = ""
+            for line in retriever.gen():
+                if "source" in line:
+                    source_log_docs.append(line["source"])
+                elif "answer" in line:
+                    response_full += line["answer"]
+
+            if data.get("isNoneDoc"):
+                for doc in source_log_docs:
+                    doc["source"] = "None"
+
+            llm = LLMCreator.create_llm(
+                settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
+            )
+
+            result = {"answer": response_full, "sources": source_log_docs}
+            result["conversation_id"] = str(
+                save_conversation(
+                    conversation_id, question, response_full, source_log_docs, llm
+                )
+            )
+            retriever_params = retriever.get_params()
+            user_logs_collection.insert_one(
+                {
+                    "action": "api_answer",
+                    "level": "info",
+                    "user": "local",
+                    "api_key": user_api_key,
+                    "question": question,
+                    "response": response_full,
+                    "sources": source_log_docs,
+                    "retriever_params": retriever_params,
+                    "timestamp": datetime.datetime.now(datetime.timezone.utc),
+                }
+            )
+
+        except Exception as e:
+            current_app.logger.error(
+                f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return bad_request(500, str(e))
+
+        return make_response(result, 200)
+
+
+@answer_ns.route("/api/search")
+class Search(Resource):
+    search_model = api.model(
+        "SearchModel",
+        {
+            "question": fields.String(
+                required=True, description="The question to search"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "api_key": fields.String(
+                required=False, description="API key for authentication"
+            ),
+            "active_docs": fields.String(
+                required=False, description="Active documents for retrieval"
+            ),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "token_limit": fields.Integer(
+                required=False, description="Limit for tokens"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
+    )
+
+    @api.expect(search_model)
+    @api.doc(
+        description="Search for relevant documents based on the question and retriever"
+    )
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                source = {"active_docs": data_key.get("source")}
+                user_api_key = data["api_key"]
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                user_api_key = None
+            else:
+                source = {}
+                user_api_key = None
+
+            current_app.logger.info(
+                f"/api/answer - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=[],
+                prompt="default",
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            docs = retriever.search()
+            retriever_params = retriever.get_params()
+
+            user_logs_collection.insert_one(
+                {
+                    "action": "api_search",
+                    "level": "info",
+                    "user": "local",
+                    "api_key": user_api_key,
+                    "question": question,
+                    "sources": docs,
+                    "retriever_params": retriever_params,
+                    "timestamp": datetime.datetime.now(datetime.timezone.utc),
+                }
+            )
+
+            if data.get("isNoneDoc"):
+                for doc in docs:
+                    doc["source"] = "None"
+
+        except Exception as e:
+            current_app.logger.error(
+                f"/api/search - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return bad_request(500, str(e))
+
+        return make_response(docs, 200)

application/api/internal/routes.py

@@ -1,20 +1,25 @@
 import os
 import datetime
 from flask import Blueprint, request, send_from_directory
-from pymongo import MongoClient
 from werkzeug.utils import secure_filename
+from bson.objectid import ObjectId
 
-
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
-mongo = MongoClient(settings.MONGO_URI)
+
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+
+internal = Blueprint("internal", __name__)
 
 
-internal = Blueprint('internal', __name__)
 @internal.route("/api/download", methods=["get"])
 def download_file():
     user = secure_filename(request.args.get("user"))
@@ -24,7 +29,6 @@ def download_file():
     return send_from_directory(save_dir, filename, as_attachment=True)
 
 
-
 @internal.route("/api/upload_index", methods=["POST"])
 def upload_index_files():
     """Upload two files(index.faiss, index.pkl) to the user's folder."""
@@ -34,7 +38,14 @@ def upload_index_files():
     if "name" not in request.form:
         return {"status": "no name"}
     job_name = secure_filename(request.form["name"])
-    save_dir = os.path.join(current_dir, "indexes", user, job_name)
+    tokens = secure_filename(request.form["tokens"])
+    retriever = secure_filename(request.form["retriever"])
+    id = secure_filename(request.form["id"])
+    type = secure_filename(request.form["type"])
+    remote_data = request.form["remote_data"] if "remote_data" in request.form else None
+    sync_frequency = secure_filename(request.form["sync_frequency"]) if "sync_frequency" in request.form else None
+
+    save_dir = os.path.join(current_dir, "indexes", str(id))
     if settings.VECTOR_STORE == "faiss":
         if "file_faiss" not in request.files:
             print("No file part")
@@ -49,21 +60,45 @@ def upload_index_files():
         if file_pkl.filename == "":
             return {"status": "no file name"}
         # saves index files
-        
+
         if not os.path.exists(save_dir):
             os.makedirs(save_dir)
         file_faiss.save(os.path.join(save_dir, "index.faiss"))
         file_pkl.save(os.path.join(save_dir, "index.pkl"))
-    # create entry in vectors_collection
-    vectors_collection.insert_one(
-        {
-            "user": user,
-            "name": job_name,
-            "language": job_name,
-            "location": save_dir,
-            "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
-            "model": settings.EMBEDDINGS_NAME,
-            "type": "local",
-        }
-    )
-    return {"status": "ok"}
+
+    existing_entry = sources_collection.find_one({"_id": ObjectId(id)})
+    if existing_entry:
+        sources_collection.update_one(
+            {"_id": ObjectId(id)},
+            {
+                "$set": {
+                    "user": user,
+                    "name": job_name,
+                    "language": job_name,
+                    "date": datetime.datetime.now(),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "type": type,
+                    "tokens": tokens,
+                    "retriever": retriever,
+                    "remote_data": remote_data,
+                    "sync_frequency": sync_frequency,
+                }
+            },
+        )
+    else:
+        sources_collection.insert_one(
+            {
+                "_id": ObjectId(id),
+                "user": user,
+                "name": job_name,
+                "language": job_name,
+                "date": datetime.datetime.now(),
+                "model": settings.EMBEDDINGS_NAME,
+                "type": type,
+                "tokens": tokens,
+                "retriever": retriever,
+                "remote_data": remote_data,
+                "sync_frequency": sync_frequency,
+            }
+        )
+    return {"status": "ok"}

application/api/user/tasks.py

@@ -1,7 +1,38 @@
-from application.worker import ingest_worker
-from application.celery import celery
+from datetime import timedelta
+
+from application.celery_init import celery
+from application.worker import ingest_worker, remote_worker, sync_worker
+
 
 @celery.task(bind=True)
 def ingest(self, directory, formats, name_job, filename, user):
     resp = ingest_worker(self, directory, formats, name_job, filename, user)
     return resp
+
+
+@celery.task(bind=True)
+def ingest_remote(self, source_data, job_name, user, loader):
+    resp = remote_worker(self, source_data, job_name, user, loader)
+    return resp
+
+
+@celery.task(bind=True)
+def schedule_syncs(self, frequency):
+    resp = sync_worker(self, frequency)
+    return resp
+
+
+@celery.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+    sender.add_periodic_task(
+        timedelta(days=1),
+        schedule_syncs.s("daily"),
+    )
+    sender.add_periodic_task(
+        timedelta(weeks=1),
+        schedule_syncs.s("weekly"),
+    )
+    sender.add_periodic_task(
+        timedelta(days=30),
+        schedule_syncs.s("monthly"),
+    )

application/app.py

@@ -1,17 +1,23 @@
 import platform
+
 import dotenv
-from application.celery import celery
-from flask import Flask, request, redirect
-from application.core.settings import settings
-from application.api.user.routes import user
+from flask import Flask, redirect, request
+
 from application.api.answer.routes import answer
 from application.api.internal.routes import internal
+from application.api.user.routes import user
+from application.celery_init import celery
+from application.core.logging_config import setup_logging
+from application.core.settings import settings
+from application.extensions import api
 
 if platform.system() == "Windows":
     import pathlib
+
     pathlib.PosixPath = pathlib.WindowsPath
 
 dotenv.load_dotenv()
+setup_logging()
 
 app = Flask(__name__)
 app.register_blueprint(user)
@@ -21,16 +27,19 @@ app.config.update(
     UPLOAD_FOLDER="inputs",
     CELERY_BROKER_URL=settings.CELERY_BROKER_URL,
     CELERY_RESULT_BACKEND=settings.CELERY_RESULT_BACKEND,
-    MONGO_URI=settings.MONGO_URI
+    MONGO_URI=settings.MONGO_URI,
 )
 celery.config_from_object("application.celeryconfig")
+api.init_app(app)
+
 
 @app.route("/")
 def home():
-    if request.remote_addr in ('0.0.0.0', '127.0.0.1', 'localhost', '172.18.0.1'):
-        return redirect('http://localhost:5173')
+    if request.remote_addr in ("0.0.0.0", "127.0.0.1", "localhost", "172.18.0.1"):
+        return redirect("http://localhost:5173")
     else:
-        return 'Welcome to DocsGPT Backend!'
+        return "Welcome to DocsGPT Backend!"
+
 
 @app.after_request
 def after_request(response):
@@ -39,6 +48,6 @@ def after_request(response):
     response.headers.add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS")
     return response
 
-if __name__ == "__main__":
-    app.run(debug=True, port=7091)
 
+if __name__ == "__main__":
+    app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)

application/cache.py

@@ -0,0 +1,93 @@
+import redis
+import time
+import json
+import logging
+from threading import Lock
+from application.core.settings import settings
+from application.utils import get_hash
+
+logger = logging.getLogger(__name__)
+
+_redis_instance = None
+_instance_lock = Lock()
+
+def get_redis_instance():
+    global _redis_instance
+    if _redis_instance is None:
+        with _instance_lock:
+            if _redis_instance is None:
+                try:
+                    _redis_instance = redis.Redis.from_url(settings.CACHE_REDIS_URL, socket_connect_timeout=2)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+                    _redis_instance = None
+    return _redis_instance
+
+def gen_cache_key(*messages, model="docgpt"):
+    if not all(isinstance(msg, dict) for msg in messages):
+        raise ValueError("All messages must be dictionaries.")
+    messages_str = json.dumps(list(messages), sort_keys=True)
+    combined = f"{model}_{messages_str}"
+    cache_key = get_hash(combined)
+    return cache_key
+
+def gen_cache(func):
+    def wrapper(self, model, messages, *args, **kwargs):
+        try:
+            cache_key = gen_cache_key(*messages)
+            redis_client = get_redis_instance()
+            if redis_client:
+                try:
+                    cached_response = redis_client.get(cache_key)
+                    if cached_response:
+                        return cached_response.decode('utf-8')
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            result = func(self, model, messages, *args, **kwargs)
+            if redis_client:
+                try:
+                    redis_client.set(cache_key, result, ex=1800)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            return result
+        except ValueError as e:
+            logger.error(e)
+            return "Error: No user message found in the conversation to generate a cache key."
+    return wrapper
+
+def stream_cache(func):
+    def wrapper(self, model, messages, stream, *args, **kwargs):
+        cache_key = gen_cache_key(*messages)
+        logger.info(f"Stream cache key: {cache_key}")
+        
+        redis_client = get_redis_instance()
+        if redis_client:
+            try:
+                cached_response = redis_client.get(cache_key)
+                if cached_response:
+                    logger.info(f"Cache hit for stream key: {cache_key}")
+                    cached_response = json.loads(cached_response.decode('utf-8'))
+                    for chunk in cached_response:
+                        yield chunk
+                        time.sleep(0.03)
+                    return
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+
+        result = func(self, model, messages, stream, *args, **kwargs)
+        stream_cache_data = []
+        
+        for chunk in result:
+            stream_cache_data.append(chunk)
+            yield chunk
+        
+        if redis_client:
+            try:
+                redis_client.set(cache_key, json.dumps(stream_cache_data), ex=1800)
+                logger.info(f"Stream cache saved for key: {cache_key}")
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+        
+    return wrapper

application/celery_init.py

@@ -1,9 +1,15 @@
 from celery import Celery
 from application.core.settings import settings
+from celery.signals import setup_logging
 
 def make_celery(app_name=__name__):
     celery = Celery(app_name, broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND)
     celery.conf.update(settings)
     return celery
 
+@setup_logging.connect
+def config_loggers(*args, **kwargs):
+    from application.core.logging_config import setup_logging
+    setup_logging()
+
 celery = make_celery()

application/core/logging_config.py

@@ -0,0 +1,22 @@
+from logging.config import dictConfig
+
+def setup_logging():
+    dictConfig({
+        'version': 1,
+        'formatters': {
+            'default': {
+                'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+            }
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "stream": "ext://sys.stdout",
+                "formatter": "default",
+            }
+        },
+        'root': {
+            'level': 'INFO',
+            'handlers': ['console'],
+        },
+    })

application/core/mongo_db.py

@@ -0,0 +1,24 @@
+from application.core.settings import settings
+from pymongo import MongoClient
+
+
+class MongoDB:
+    _client = None
+
+    @classmethod
+    def get_client(cls):
+        """
+        Get the MongoDB client instance, creating it if necessary.
+        """
+        if cls._client is None:
+            cls._client = MongoClient(settings.MONGO_URI)
+        return cls._client
+
+    @classmethod
+    def close_client(cls):
+        """
+        Close the MongoDB client connection.
+        """
+        if cls._client is not None:
+            cls._client.close()
+            cls._client = None

application/core/settings.py

@@ -1,42 +1,81 @@
 from pathlib import Path
+from typing import Optional
 import os
 
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings
+
 current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
 class Settings(BaseSettings):
-    LLM_NAME: str = "openai"
-    EMBEDDINGS_NAME: str = "openai_text-embedding-ada-002"
+    LLM_NAME: str = "docsgpt"
+    MODEL_NAME: Optional[str] = None # if LLM_NAME is openai, MODEL_NAME can be gpt-4 or gpt-3.5-turbo
+    EMBEDDINGS_NAME: str = "huggingface_sentence-transformers/all-mpnet-base-v2"
     CELERY_BROKER_URL: str = "redis://localhost:6379/0"
     CELERY_RESULT_BACKEND: str = "redis://localhost:6379/1"
     MONGO_URI: str = "mongodb://localhost:27017/docsgpt"
     MODEL_PATH: str = os.path.join(current_dir, "models/docsgpt-7b-f16.gguf")
-    TOKENS_MAX_HISTORY: int = 150
+    DEFAULT_MAX_HISTORY: int = 150
+    MODEL_TOKEN_LIMITS: dict = {"gpt-3.5-turbo": 4096, "claude-2": 1e5}
     UPLOAD_FOLDER: str = "inputs"
-    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch"
+    VECTOR_STORE: str = "faiss" #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb"
+    RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
+
+    # LLM Cache
+    CACHE_REDIS_URL: str = "redis://localhost:6379/2"
 
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
 
-    API_KEY: str = None  # LLM api key
-    EMBEDDINGS_KEY: str = None  # api key for embeddings (if using openai, just copy API_KEY)
-    OPENAI_API_BASE: str = None  # azure openai api base url
-    OPENAI_API_VERSION: str = None  # azure openai api version
-    AZURE_DEPLOYMENT_NAME: str = None  # azure deployment name for answering
-    AZURE_EMBEDDINGS_DEPLOYMENT_NAME: str = None  # azure deployment name for embeddings
+    API_KEY: Optional[str] = None  # LLM api key
+    EMBEDDINGS_KEY: Optional[str] = None  # api key for embeddings (if using openai, just copy API_KEY)
+    OPENAI_API_BASE: Optional[str] = None  # azure openai api base url
+    OPENAI_API_VERSION: Optional[str] = None  # azure openai api version
+    AZURE_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for answering
+    AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for embeddings
+    OPENAI_BASE_URL: Optional[str] = None # openai base url for open ai compatable models
 
     # elasticsearch
-    ELASTIC_CLOUD_ID: str = None # cloud id for elasticsearch
-    ELASTIC_USERNAME: str = None # username for elasticsearch
-    ELASTIC_PASSWORD: str = None # password for elasticsearch
-    ELASTIC_URL: str = None # url for elasticsearch
-    ELASTIC_INDEX: str = "docsgpt" # index name for elasticsearch
+    ELASTIC_CLOUD_ID: Optional[str] = None  # cloud id for elasticsearch
+    ELASTIC_USERNAME: Optional[str] = None  # username for elasticsearch
+    ELASTIC_PASSWORD: Optional[str] = None  # password for elasticsearch
+    ELASTIC_URL: Optional[str] = None  # url for elasticsearch
+    ELASTIC_INDEX: Optional[str] = "docsgpt"  # index name for elasticsearch
 
     # SageMaker config
-    SAGEMAKER_ENDPOINT: str = None # SageMaker endpoint name
-    SAGEMAKER_REGION: str = None # SageMaker region name
-    SAGEMAKER_ACCESS_KEY: str = None # SageMaker access key
-    SAGEMAKER_SECRET_KEY: str = None # SageMaker secret key
+    SAGEMAKER_ENDPOINT: Optional[str] = None  # SageMaker endpoint name
+    SAGEMAKER_REGION: Optional[str] = None  # SageMaker region name
+    SAGEMAKER_ACCESS_KEY: Optional[str] = None  # SageMaker access key
+    SAGEMAKER_SECRET_KEY: Optional[str] = None  # SageMaker secret key
+
+    # prem ai project id
+    PREMAI_PROJECT_ID: Optional[str] = None
+
+    # Qdrant vectorstore config
+    QDRANT_COLLECTION_NAME: Optional[str] = "docsgpt"
+    QDRANT_LOCATION: Optional[str] = None
+    QDRANT_URL: Optional[str] = None
+    QDRANT_PORT: Optional[int] = 6333
+    QDRANT_GRPC_PORT: int = 6334
+    QDRANT_PREFER_GRPC: bool = False
+    QDRANT_HTTPS: Optional[bool] = None
+    QDRANT_API_KEY: Optional[str] = None
+    QDRANT_PREFIX: Optional[str] = None
+    QDRANT_TIMEOUT: Optional[float] = None
+    QDRANT_HOST: Optional[str] = None
+    QDRANT_PATH: Optional[str] = None
+    QDRANT_DISTANCE_FUNC: str = "Cosine"
+
+    # Milvus vectorstore config
+    MILVUS_COLLECTION_NAME: Optional[str] = "docsgpt"
+    MILVUS_URI: Optional[str] = "./milvus_local.db"   # milvus lite version as default
+    MILVUS_TOKEN: Optional[str] = ""
+
+    # LanceDB vectorstore config
+    LANCEDB_PATH: str = "/tmp/lancedb"  # Path where LanceDB stores its local data
+    LANCEDB_TABLE_NAME: Optional[str] = "docsgpts"  # Name of the table to use for storing vectors
+    BRAVE_SEARCH_API_KEY: Optional[str] = None
+
+    FLASK_DEBUG_MODE: bool = False
 
 
 path = Path(__file__).parent.parent.absolute()

application/extensions.py

@@ -0,0 +1,7 @@
+from flask_restx import Api
+
+api = Api(
+    version="1.0",
+    title="DocsGPT API",
+    description="API for DocsGPT",
+)

application/llm/anthropic.py

@@ -1,21 +1,29 @@
 from application.llm.base import BaseLLM
 from application.core.settings import settings
 
+
 class AnthropicLLM(BaseLLM):
 
-    def __init__(self, api_key=None):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
         from anthropic import Anthropic, HUMAN_PROMPT, AI_PROMPT
-        self.api_key = api_key or settings.ANTHROPIC_API_KEY  # If not provided, use a default from settings
+
+        super().__init__(*args, **kwargs)
+        self.api_key = (
+            api_key or settings.ANTHROPIC_API_KEY
+        )  # If not provided, use a default from settings
+        self.user_api_key = user_api_key
         self.anthropic = Anthropic(api_key=self.api_key)
         self.HUMAN_PROMPT = HUMAN_PROMPT
         self.AI_PROMPT = AI_PROMPT
 
-    def gen(self, model, messages, engine=None, max_tokens=300, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(
+        self, baseself, model, messages, stream=False, max_tokens=300, **kwargs
+    ):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Context \n {context} \n ### Question \n {user_question}"
         if stream:
-            return self.gen_stream(model, prompt, max_tokens, **kwargs)
+            return self.gen_stream(model, prompt, stream, max_tokens, **kwargs)
 
         completion = self.anthropic.completions.create(
             model=model,
@@ -25,9 +33,11 @@ class AnthropicLLM(BaseLLM):
         )
         return completion.completion
 
-    def gen_stream(self, model, messages, engine=None, max_tokens=300, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen_stream(
+        self, baseself, model, messages, stream=True, max_tokens=300, **kwargs
+    ):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Context \n {context} \n ### Question \n {user_question}"
         stream_response = self.anthropic.completions.create(
             model=model,
@@ -37,4 +47,4 @@ class AnthropicLLM(BaseLLM):
         )
 
         for completion in stream_response:
-            yield completion.completion
+            yield completion.completion

application/llm/base.py

@@ -1,14 +1,29 @@
 from abc import ABC, abstractmethod
+from application.usage import gen_token_usage, stream_token_usage
+from application.cache import stream_cache, gen_cache
 
 
 class BaseLLM(ABC):
     def __init__(self):
-        pass
+        self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
+
+    def _apply_decorator(self, method, decorators, *args, **kwargs):
+        for decorator in decorators:
+            method = decorator(method)
+        return method(self, *args, **kwargs)
 
     @abstractmethod
-    def gen(self, *args, **kwargs):
+    def _raw_gen(self, model, messages, stream, *args, **kwargs):
         pass
 
+    def gen(self, model, messages, stream=False, *args, **kwargs):
+        decorators = [gen_token_usage, gen_cache]
+        return self._apply_decorator(self._raw_gen, decorators=decorators, model=model, messages=messages, stream=stream, *args, **kwargs)
+
     @abstractmethod
-    def gen_stream(self, *args, **kwargs):
+    def _raw_gen_stream(self, model, messages, stream, *args, **kwargs):
         pass
+
+    def gen_stream(self, model, messages, stream=True, *args, **kwargs):
+        decorators = [stream_cache, stream_token_usage]
+        return self._apply_decorator(self._raw_gen_stream, decorators=decorators, model=model, messages=messages, stream=stream, *args, **kwargs)

application/llm/docsgpt_provider.py

@@ -0,0 +1,44 @@
+from application.llm.base import BaseLLM
+import json
+import requests
+
+
+class DocsGPTAPILLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.endpoint = "https://llm.docsgpt.co.uk"
+
+    def _raw_gen(self, baseself, model, messages, stream=False, *args, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
+        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
+
+        response = requests.post(
+            f"{self.endpoint}/answer", json={"prompt": prompt, "max_new_tokens": 30}
+        )
+        response_clean = response.json()["a"].replace("###", "")
+
+        return response_clean
+
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, *args, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
+        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
+
+        # send prompt to endpoint /stream
+        response = requests.post(
+            f"{self.endpoint}/stream",
+            json={"prompt": prompt, "max_new_tokens": 256},
+            stream=True,
+        )
+
+        for line in response.iter_lines():
+            if line:
+                # data = json.loads(line)
+                data_str = line.decode("utf-8")
+                if data_str.startswith("data: "):
+                    data = json.loads(data_str[6:])
+                    yield data["a"]

application/llm/groq.py

@@ -0,0 +1,45 @@
+from application.llm.base import BaseLLM
+
+
+
+class GroqLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from openai import OpenAI
+
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        for line in response:
+            # import sys
+            # print(line.choices[0].delta.content, file=sys.stderr)
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content

application/llm/huggingface.py

@@ -1,44 +1,68 @@
 from application.llm.base import BaseLLM
 
+
 class HuggingFaceLLM(BaseLLM):
 
-    def __init__(self, api_key, llm_name='Arc53/DocsGPT-7B',q=False):
+    def __init__(
+        self,
+        api_key=None,
+        user_api_key=None,
+        llm_name="Arc53/DocsGPT-7B",
+        q=False,
+        *args,
+        **kwargs,
+    ):
         global hf
-        
+
         from langchain.llms import HuggingFacePipeline
+
         if q:
             import torch
-            from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig
+            from transformers import (
+                AutoModelForCausalLM,
+                AutoTokenizer,
+                pipeline,
+                BitsAndBytesConfig,
+            )
+
             tokenizer = AutoTokenizer.from_pretrained(llm_name)
             bnb_config = BitsAndBytesConfig(
-                            load_in_4bit=True,
-                            bnb_4bit_use_double_quant=True,
-                            bnb_4bit_quant_type="nf4",
-                            bnb_4bit_compute_dtype=torch.bfloat16
-                        )
-            model = AutoModelForCausalLM.from_pretrained(llm_name,quantization_config=bnb_config)
+                load_in_4bit=True,
+                bnb_4bit_use_double_quant=True,
+                bnb_4bit_quant_type="nf4",
+                bnb_4bit_compute_dtype=torch.bfloat16,
+            )
+            model = AutoModelForCausalLM.from_pretrained(
+                llm_name, quantization_config=bnb_config
+            )
         else:
             from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
             tokenizer = AutoTokenizer.from_pretrained(llm_name)
             model = AutoModelForCausalLM.from_pretrained(llm_name)
-        
+
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
         pipe = pipeline(
-            "text-generation", model=model,
-            tokenizer=tokenizer, max_new_tokens=2000,
-            device_map="auto", eos_token_id=tokenizer.eos_token_id
+            "text-generation",
+            model=model,
+            tokenizer=tokenizer,
+            max_new_tokens=2000,
+            device_map="auto",
+            eos_token_id=tokenizer.eos_token_id,
         )
         hf = HuggingFacePipeline(pipeline=pipe)
 
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
 
         result = hf(prompt)
 
         return result.content
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
 
         raise NotImplementedError("HuggingFaceLLM Streaming is not implemented yet.")
-

application/llm/llama_cpp.py

@@ -1,39 +1,55 @@
 from application.llm.base import BaseLLM
 from application.core.settings import settings
+import threading
+
+class LlamaSingleton:
+    _instances = {}
+    _lock = threading.Lock()  # Add a lock for thread synchronization
+
+    @classmethod
+    def get_instance(cls, llm_name):
+        if llm_name not in cls._instances:
+            try:
+                from llama_cpp import Llama
+            except ImportError:
+                raise ImportError(
+                    "Please install llama_cpp using pip install llama-cpp-python"
+                )
+            cls._instances[llm_name] = Llama(model_path=llm_name, n_ctx=2048)
+        return cls._instances[llm_name]
+
+    @classmethod
+    def query_model(cls, llm, prompt, **kwargs):
+        with cls._lock:
+            return llm(prompt, **kwargs)
+
 
 class LlamaCpp(BaseLLM):
+    def __init__(
+        self,
+        api_key=None,
+        user_api_key=None,
+        llm_name=settings.MODEL_PATH,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.llama = LlamaSingleton.get_instance(llm_name)
 
-    def __init__(self, api_key, llm_name=settings.MODEL_PATH, **kwargs):
-        global llama
-        try:
-            from llama_cpp import Llama
-        except ImportError:
-            raise ImportError("Please install llama_cpp using pip install llama-cpp-python")
-
-        llama = Llama(model_path=llm_name, n_ctx=2048)
-
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
+        result = LlamaSingleton.query_model(self.llama, prompt, max_tokens=150, echo=False)
+        return result["choices"][0]["text"].split("### Answer \n")[-1]
 
-        result = llama(prompt, max_tokens=150, echo=False)
-
-        # import sys
-        # print(result['choices'][0]['text'].split('### Answer \n')[-1], file=sys.stderr)
-        
-        return result['choices'][0]['text'].split('### Answer \n')[-1]
-
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
-        result = llama(prompt, max_tokens=150, echo=False, stream=stream)
-
-        # import sys
-        # print(list(result), file=sys.stderr)
-
+        result = LlamaSingleton.query_model(self.llama, prompt, max_tokens=150, echo=False, stream=stream)
         for item in result:
-            for choice in item['choices']:
-                yield choice['text']
+            for choice in item["choices"]:
+                yield choice["text"]

application/llm/llm_creator.py

@@ -1,24 +1,29 @@
+from application.llm.groq import GroqLLM
 from application.llm.openai import OpenAILLM, AzureOpenAILLM
 from application.llm.sagemaker import SagemakerAPILLM
 from application.llm.huggingface import HuggingFaceLLM
 from application.llm.llama_cpp import LlamaCpp
 from application.llm.anthropic import AnthropicLLM
-
+from application.llm.docsgpt_provider import DocsGPTAPILLM
+from application.llm.premai import PremAILLM
 
 
 class LLMCreator:
     llms = {
-        'openai': OpenAILLM,
-        'azure_openai': AzureOpenAILLM,
-        'sagemaker': SagemakerAPILLM,
-        'huggingface': HuggingFaceLLM,
-        'llama.cpp': LlamaCpp,
-        'anthropic': AnthropicLLM
+        "openai": OpenAILLM,
+        "azure_openai": AzureOpenAILLM,
+        "sagemaker": SagemakerAPILLM,
+        "huggingface": HuggingFaceLLM,
+        "llama.cpp": LlamaCpp,
+        "anthropic": AnthropicLLM,
+        "docsgpt": DocsGPTAPILLM,
+        "premai": PremAILLM,
+        "groq": GroqLLM
     }
 
     @classmethod
-    def create_llm(cls, type, *args, **kwargs):
+    def create_llm(cls, type, api_key, user_api_key, *args, **kwargs):
         llm_class = cls.llms.get(type.lower())
         if not llm_class:
             raise ValueError(f"No LLM class found for type {type}")
-        return llm_class(*args, **kwargs)
+        return llm_class(api_key, user_api_key, *args, **kwargs)

application/llm/openai.py

@@ -1,57 +1,73 @@
 from application.llm.base import BaseLLM
 from application.core.settings import settings
 
+
+
 class OpenAILLM(BaseLLM):
 
-    def __init__(self, api_key):
-        global openai
-        import openai
-        openai.api_key = api_key
-        self.api_key = api_key  # Save the API key to be used later
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from openai import OpenAI
 
-    def _get_openai(self):
-        # Import openai when needed
-        import openai
-        # Set the API key every time you import openai
-        openai.api_key = self.api_key
-        return openai
+        super().__init__(*args, **kwargs)
+        if settings.OPENAI_BASE_URL:
+            self.client = OpenAI(
+                api_key=api_key,
+                base_url=settings.OPENAI_BASE_URL
+            )
+        else:
+            self.client = OpenAI(api_key=api_key)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
 
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        response = openai.ChatCompletion.create(
-            model=model,
-            engine=engine,
-            messages=messages,
-            stream=stream,
-            **kwargs
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        engine=settings.AZURE_DEPLOYMENT_NAME,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
         )
 
-        return response["choices"][0]["message"]["content"]
+        return response.choices[0].message.content
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        response = openai.ChatCompletion.create(
-            model=model,
-            engine=engine,
-            messages=messages,
-            stream=stream,
-            **kwargs
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        engine=settings.AZURE_DEPLOYMENT_NAME,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
         )
 
         for line in response:
-            if "content" in line["choices"][0]["delta"]:
-                yield line["choices"][0]["delta"]["content"]
+            # import sys
+            # print(line.choices[0].delta.content, file=sys.stderr)
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content
 
 
 class AzureOpenAILLM(OpenAILLM):
 
-    def __init__(self, openai_api_key, openai_api_base, openai_api_version, deployment_name):
+    def __init__(
+        self, openai_api_key, openai_api_base, openai_api_version, deployment_name
+    ):
         super().__init__(openai_api_key)
-        self.api_base = settings.OPENAI_API_BASE,
-        self.api_version = settings.OPENAI_API_VERSION,
-        self.deployment_name = settings.AZURE_DEPLOYMENT_NAME,
+        self.api_base = (settings.OPENAI_API_BASE,)
+        self.api_version = (settings.OPENAI_API_VERSION,)
+        self.deployment_name = (settings.AZURE_DEPLOYMENT_NAME,)
+        from openai import AzureOpenAI
 
-    def _get_openai(self):
-        openai = super()._get_openai()
-        openai.api_base = self.api_base
-        openai.api_version = self.api_version
-        openai.api_type = "azure"
-        return openai
+        self.client = AzureOpenAI(
+            api_key=openai_api_key,
+            api_version=settings.OPENAI_API_VERSION,
+            api_base=settings.OPENAI_API_BASE,
+            deployment_name=settings.AZURE_DEPLOYMENT_NAME,
+        )

application/llm/premai.py

@@ -0,0 +1,38 @@
+from application.llm.base import BaseLLM
+from application.core.settings import settings
+
+
+class PremAILLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from premai import Prem
+
+        super().__init__(*args, **kwargs)
+        self.client = Prem(api_key=api_key)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.project_id = settings.PREMAI_PROJECT_ID
+
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        response = self.client.chat.completions.create(
+            model=model,
+            project_id=self.project_id,
+            messages=messages,
+            stream=stream,
+            **kwargs
+        )
+
+        return response.choices[0].message["content"]
+
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+        response = self.client.chat.completions.create(
+            model=model,
+            project_id=self.project_id,
+            messages=messages,
+            stream=stream,
+            **kwargs
+        )
+
+        for line in response:
+            if line.choices[0].delta["content"] is not None:
+                yield line.choices[0].delta["content"]

application/llm/sagemaker.py

@@ -4,11 +4,10 @@ import json
 import io
 
 
-
 class LineIterator:
     """
-    A helper class for parsing the byte stream input. 
-    
+    A helper class for parsing the byte stream input.
+
     The output of the model will be in the following format:
     ```
     b'{"outputs": [" a"]}\n'
@@ -16,21 +15,21 @@ class LineIterator:
     b'{"outputs": [" problem"]}\n'
     ...
     ```
-    
-    While usually each PayloadPart event from the event stream will contain a byte array 
+
+    While usually each PayloadPart event from the event stream will contain a byte array
     with a full json, this is not guaranteed and some of the json objects may be split across
     PayloadPart events. For example:
     ```
     {'PayloadPart': {'Bytes': b'{"outputs": '}}
     {'PayloadPart': {'Bytes': b'[" problem"]}\n'}}
     ```
-    
+
     This class accounts for this by concatenating bytes written via the 'write' function
     and then exposing a method which will return lines (ending with a '\n' character) within
-    the buffer via the 'scan_lines' function. It maintains the position of the last read 
-    position to ensure that previous bytes are not exposed again. 
+    the buffer via the 'scan_lines' function. It maintains the position of the last read
+    position to ensure that previous bytes are not exposed again.
     """
-    
+
     def __init__(self, stream):
         self.byte_iterator = iter(stream)
         self.buffer = io.BytesIO()
@@ -43,7 +42,7 @@ class LineIterator:
         while True:
             self.buffer.seek(self.read_pos)
             line = self.buffer.readline()
-            if line and line[-1] == ord('\n'):
+            if line and line[-1] == ord("\n"):
                 self.read_pos += len(line)
                 return line[:-1]
             try:
@@ -52,33 +51,35 @@ class LineIterator:
                 if self.read_pos < self.buffer.getbuffer().nbytes:
                     continue
                 raise
-            if 'PayloadPart' not in chunk:
-                print('Unknown event type:' + chunk)
+            if "PayloadPart" not in chunk:
+                print("Unknown event type:" + chunk)
                 continue
             self.buffer.seek(0, io.SEEK_END)
-            self.buffer.write(chunk['PayloadPart']['Bytes'])
+            self.buffer.write(chunk["PayloadPart"]["Bytes"])
+
 
 class SagemakerAPILLM(BaseLLM):
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
         import boto3
+
         runtime = boto3.client(
-            'runtime.sagemaker',
-            aws_access_key_id='xxx',
-            aws_secret_access_key='xxx',
-            region_name='us-west-2'
+            "runtime.sagemaker",
+            aws_access_key_id="xxx",
+            aws_secret_access_key="xxx",
+            region_name="us-west-2",
         )
 
-        
-        self.endpoint =  settings.SAGEMAKER_ENDPOINT
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.endpoint = settings.SAGEMAKER_ENDPOINT
         self.runtime = runtime
 
-
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-    
 
         # Construct payload for endpoint
         payload = {
@@ -89,25 +90,25 @@ class SagemakerAPILLM(BaseLLM):
                 "temperature": 0.1,
                 "max_new_tokens": 30,
                 "repetition_penalty": 1.03,
-                "stop": ["</s>", "###"]
-            }
+                "stop": ["</s>", "###"],
+            },
         }
-        body_bytes = json.dumps(payload).encode('utf-8')
+        body_bytes = json.dumps(payload).encode("utf-8")
 
         # Invoke the endpoint
-        response = self.runtime.invoke_endpoint(EndpointName=self.endpoint,
-                                        ContentType='application/json',
-                                        Body=body_bytes)
-        result = json.loads(response['Body'].read().decode())
+        response = self.runtime.invoke_endpoint(
+            EndpointName=self.endpoint, ContentType="application/json", Body=body_bytes
+        )
+        result = json.loads(response["Body"].read().decode())
         import sys
-        print(result[0]['generated_text'], file=sys.stderr)
-        return result[0]['generated_text'][len(prompt):]
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+        print(result[0]["generated_text"], file=sys.stderr)
+        return result[0]["generated_text"][len(prompt) :]
+
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-    
 
         # Construct payload for endpoint
         payload = {
@@ -118,22 +119,22 @@ class SagemakerAPILLM(BaseLLM):
                 "temperature": 0.1,
                 "max_new_tokens": 512,
                 "repetition_penalty": 1.03,
-                "stop": ["</s>", "###"]
-            }
+                "stop": ["</s>", "###"],
+            },
         }
-        body_bytes = json.dumps(payload).encode('utf-8')
+        body_bytes = json.dumps(payload).encode("utf-8")
 
         # Invoke the endpoint
-        response = self.runtime.invoke_endpoint_with_response_stream(EndpointName=self.endpoint,
-                                        ContentType='application/json',
-                                        Body=body_bytes)
-        #result = json.loads(response['Body'].read().decode())
-        event_stream = response['Body']
-        start_json = b'{'
+        response = self.runtime.invoke_endpoint_with_response_stream(
+            EndpointName=self.endpoint, ContentType="application/json", Body=body_bytes
+        )
+        # result = json.loads(response['Body'].read().decode())
+        event_stream = response["Body"]
+        start_json = b"{"
         for line in LineIterator(event_stream):
-            if line != b'' and start_json in line:
-                #print(line)
-                data = json.loads(line[line.find(start_json):].decode('utf-8'))
-                if data['token']['text'] not in ["</s>", "###"]:
-                    print(data['token']['text'],end='')
-                    yield data['token']['text']
+            if line != b"" and start_json in line:
+                # print(line)
+                data = json.loads(line[line.find(start_json) :].decode("utf-8"))
+                if data["token"]["text"] not in ["</s>", "###"]:
+                    print(data["token"]["text"], end="")
+                    yield data["token"]["text"]

application/parser/file/bulk.py

@@ -10,18 +10,23 @@ from application.parser.file.epub_parser import EpubParser
 from application.parser.file.html_parser import HTMLParser
 from application.parser.file.markdown_parser import MarkdownParser
 from application.parser.file.rst_parser import RstParser
-from application.parser.file.tabular_parser import PandasCSVParser
+from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser
+from application.parser.file.json_parser import JSONParser
+from application.parser.file.pptx_parser import PPTXParser
 from application.parser.schema.base import Document
 
 DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
     ".pdf": PDFParser(),
     ".docx": DocxParser(),
     ".csv": PandasCSVParser(),
+    ".xlsx":ExcelParser(),
     ".epub": EpubParser(),
     ".md": MarkdownParser(),
     ".rst": RstParser(),
     ".html": HTMLParser(),
     ".mdx": MarkdownParser(),
+    ".json":JSONParser(),
+    ".pptx":PPTXParser(),
 }
 
 
@@ -62,7 +67,6 @@ class SimpleDirectoryReader(BaseReader):
             file_extractor: Optional[Dict[str, BaseParser]] = None,
             num_files_limit: Optional[int] = None,
             file_metadata: Optional[Callable[[str], Dict]] = None,
-            chunk_size_max: int = 2048,
     ) -> None:
         """Initialize with parameters."""
         super().__init__()
@@ -148,12 +152,24 @@ class SimpleDirectoryReader(BaseReader):
                 # do standard read
                 with open(input_file, "r", errors=self.errors) as f:
                     data = f.read()
-            if isinstance(data, List):
-                data_list.extend(data)
-            else:
-                data_list.append(str(data))
+            # Prepare metadata for this file
             if self.file_metadata is not None:
-                metadata_list.append(self.file_metadata(str(input_file)))
+                file_metadata = self.file_metadata(str(input_file))
+            else:
+                # Provide a default empty metadata
+                file_metadata = {'title': '', 'store': ''}
+                # TODO: Find a case with no metadata and check if breaks anything 
+
+            if isinstance(data, List):
+                # Extend data_list with each item in the data list
+                data_list.extend([str(d) for d in data])
+                # For each item in the data list, add the file's metadata to metadata_list
+                metadata_list.extend([file_metadata for _ in data])
+            else:
+                # Add the single piece of data to data_list
+                data_list.append(str(data))
+                # Add the file's metadata to metadata_list
+                metadata_list.append(file_metadata)
 
         if concatenate:
             return [Document("\n".join(data_list))]

application/parser/file/html_parser.py

@@ -3,7 +3,6 @@
 Contains parser for html files.
 
 """
-import re
 from pathlib import Path
 from typing import Dict, Union
 
@@ -18,66 +17,8 @@ class HTMLParser(BaseParser):
         return {}
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
-        """Parse file.
+        from langchain_community.document_loaders import BSHTMLLoader
 
-            Returns:
-            Union[str, List[str]]: a string or a List of strings.
-        """
-        try:
-            from unstructured.partition.html import partition_html
-            from unstructured.staging.base import convert_to_isd
-            from unstructured.cleaners.core import clean
-        except ImportError:
-            raise ValueError("unstructured package is required to parse HTML files.")
-
-        # Using the unstructured library to convert the html to isd format
-        # isd sample : isd = [
-        #   {"text": "My Title", "type": "Title"},
-        #   {"text": "My Narrative", "type": "NarrativeText"}
-        # ]
-        with open(file, "r", encoding="utf-8") as fp:
-            elements = partition_html(file=fp)
-            isd = convert_to_isd(elements)
-
-            # Removing non ascii charactwers from isd_el['text']
-        for isd_el in isd:
-            isd_el['text'] = isd_el['text'].encode("ascii", "ignore").decode()
-
-        # Removing all the \n characters from isd_el['text'] using regex and replace with single space
-        # Removing all the extra spaces  from isd_el['text'] using regex and replace with single space
-        for isd_el in isd:
-            isd_el['text'] = re.sub(r'\n', ' ', isd_el['text'], flags=re.MULTILINE | re.DOTALL)
-            isd_el['text'] = re.sub(r"\s{2,}", " ", isd_el['text'], flags=re.MULTILINE | re.DOTALL)
-
-        # more cleaning: extra_whitespaces, dashes, bullets, trailing_punctuation
-        for isd_el in isd:
-            clean(isd_el['text'], extra_whitespace=True, dashes=True, bullets=True, trailing_punctuation=True)
-
-        # Creating a list of all the indexes of isd_el['type'] = 'Title'
-        title_indexes = [i for i, isd_el in enumerate(isd) if isd_el['type'] == 'Title']
-
-        # Creating 'Chunks' - List of lists of strings 
-        # each list starting with isd_el['type'] = 'Title' and all the data till the next 'Title'
-        # Each Chunk can be thought of as an individual set of data, which can be sent to the model
-        # Where Each Title is grouped together with the data under it
-
-        Chunks = [[]]
-        final_chunks = list(list())
-
-        for i, isd_el in enumerate(isd):
-            if i in title_indexes:
-                Chunks.append([])
-            Chunks[-1].append(isd_el['text'])
-
-        # Removing all the chunks with sum of length of all the strings in the chunk < 25
-        # TODO: This value can be an user defined variable
-        for chunk in Chunks:
-            # sum of length of all the strings in the chunk
-            sum = 0
-            sum += len(str(chunk))
-            if sum < 25:
-                Chunks.remove(chunk)
-            else:
-                # appending all the approved chunks to final_chunks as a single string       
-                final_chunks.append(" ".join([str(item) for item in chunk]))
-        return final_chunks
+        loader = BSHTMLLoader(file)
+        data = loader.load()        
+        return data

application/parser/file/json_parser.py

@@ -0,0 +1,57 @@
+import json
+from typing import Any, Dict, List, Union
+from pathlib import Path
+
+from application.parser.file.base_parser import BaseParser
+
+class JSONParser(BaseParser):
+    r"""JSON (.json) parser.
+
+    Parses JSON files into a list of strings or a concatenated document.
+    It handles both JSON objects (dictionaries) and arrays (lists).
+
+    Args:
+        concat_rows (bool): Whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each item in the JSON.
+            True by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        json_config (dict): Options for parsing JSON. Can be used to specify options like
+        custom decoding or formatting. Set to empty dict by default.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            row_joiner: str = "\n",
+            json_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._row_joiner = row_joiner
+        self._json_config = json_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse JSON file."""
+        
+        with open(file, 'r', encoding='utf-8') as f:
+                data = json.load(f, **self._json_config)
+
+        if isinstance(data, dict):
+            data = [data]
+
+        if self._concat_rows:
+            return self._row_joiner.join([str(item) for item in data])
+        else:
+            return data

application/parser/file/pptx_parser.py

@@ -0,0 +1,75 @@
+"""PPT parser.
+Contains parsers for presentation (.pptx) files to extract slide text.
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from application.parser.file.base_parser import BaseParser
+
+class PPTXParser(BaseParser):
+    r"""PPTX (.pptx) parser for extracting text from PowerPoint slides.
+    Args:
+        concat_slides (bool): Specifies whether to concatenate all slide text into one document.
+            - If True, slide texts will be joined together as a single string.
+            - If False, each slide's text will be stored as a separate entry in a list.
+            Set to True by default.
+        slide_separator (str): Separator used to join slides' text content.
+            Only used when `concat_slides=True`. Default is "\n".
+        Refer to https://python-pptx.readthedocs.io/en/latest/ for more information.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        concat_slides: bool = True,
+        slide_separator: str = "\n",
+        **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_slides = concat_slides
+        self._slide_separator = slide_separator
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        r"""
+        Parse a .pptx file and extract text from each slide.
+        Args:
+            file (Path): Path to the .pptx file.
+            errors (str): Error handling policy ('ignore' by default).
+        Returns:
+            Union[str, List[str]]: Concatenated text if concat_slides is True,
+            otherwise a list of slide texts.
+        """
+
+        try:
+            from pptx import Presentation
+        except ImportError:
+            raise ImportError("pptx module is required to read .PPTX files.")
+
+        try:
+            presentation = Presentation(file)
+            slide_texts=[]
+
+            # Iterate over each slide in the presentation
+            for slide in presentation.slides:
+                slide_text=""
+
+                # Iterate over each shape in the slide
+                for shape in slide.shapes:
+                    # Check if the shape has a 'text' attribute and append that to the slide_text
+                    if hasattr(shape,"text"):
+                        slide_text+=shape.text
+
+                slide_texts.append(slide_text.strip())
+
+            if self._concat_slides:
+                return self._slide_separator.join(slide_texts)
+            else:
+                return slide_texts
+
+        except Exception as e:
+            raise e

application/parser/file/tabular_parser.py

@@ -113,3 +113,68 @@ class PandasCSVParser(BaseParser):
             return (self._row_joiner).join(text_list)
         else:
             return text_list
+
+
+class ExcelParser(BaseParser):
+    r"""Excel (.xlsx) parser.
+
+    Parses Excel files using Pandas `read_excel` function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_excel` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the table structure on its own.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read Excel files.")
+
+        df = pd.read_excel(file, **self._pandas_config)
+
+        text_list = df.apply(
+            lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
+        ).tolist()
+
+        if self._concat_rows:
+            return (self._row_joiner).join(text_list)
+        else:
+            return text_list

application/parser/open_ai_func.py

@@ -1,38 +1,33 @@
 import os
 
-import tiktoken
-from application.vectorstore.vector_creator import VectorCreator
-from application.core.settings import settings
 from retry import retry
 
+from application.core.settings import settings
 
-# from langchain.embeddings import HuggingFaceEmbeddings
-# from langchain.embeddings import HuggingFaceInstructEmbeddings
-# from langchain.embeddings import CohereEmbeddings
+from application.vectorstore.vector_creator import VectorCreator
 
 
-def num_tokens_from_string(string: str, encoding_name: str) -> int:
-    # Function to convert string to tokens and estimate user cost.
-    encoding = tiktoken.get_encoding(encoding_name)
-    num_tokens = len(encoding.encode(string))
-    total_price = ((num_tokens / 1000) * 0.0004)
-    return num_tokens, total_price
+# from langchain_community.embeddings import HuggingFaceEmbeddings
+# from langchain_community.embeddings import HuggingFaceInstructEmbeddings
+# from langchain_community.embeddings import CohereEmbeddings
 
 
 @retry(tries=10, delay=60)
-def store_add_texts_with_retry(store, i):
+def store_add_texts_with_retry(store, i, id):
+    # add source_id to the metadata
+    i.metadata["source_id"] = str(id)
     store.add_texts([i.page_content], metadatas=[i.metadata])
     # store_pine.add_texts([i.page_content], metadatas=[i.metadata])
 
 
-def call_openai_api(docs, folder_name, task_status):
-    # Function to create a vector store from the documents and save it to disk.
+def call_openai_api(docs, folder_name, id, task_status):
+    # Function to create a vector store from the documents and save it to disk
 
-    # create output folder if it doesn't exist
     if not os.path.exists(f"{folder_name}"):
         os.makedirs(f"{folder_name}")
 
     from tqdm import tqdm
+
     c1 = 0
     if settings.VECTOR_STORE == "faiss":
         docs_init = [docs[0]]
@@ -40,26 +35,34 @@ def call_openai_api(docs, folder_name, task_status):
 
         store = VectorCreator.create_vectorstore(
             settings.VECTOR_STORE,
-            docs_init = docs_init,
-            path=f"{folder_name}",
-            embeddings_key=os.getenv("EMBEDDINGS_KEY")
+            docs_init=docs_init,
+            source_id=f"{folder_name}",
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
         )
     else:
         store = VectorCreator.create_vectorstore(
             settings.VECTOR_STORE,
-            path=f"{folder_name}",
-            embeddings_key=os.getenv("EMBEDDINGS_KEY")
+            source_id=str(id),
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
         )
+        store.delete_index()
     # Uncomment for MPNet embeddings
     # model_name = "sentence-transformers/all-mpnet-base-v2"
     # hf = HuggingFaceEmbeddings(model_name=model_name)
     # store = FAISS.from_documents(docs_test, hf)
     s1 = len(docs)
-    for i in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs),
-                  bar_format='{l_bar}{bar}| Time Left: {remaining}'):
+    for i in tqdm(
+        docs,
+        desc="Embedding 🦖",
+        unit="docs",
+        total=len(docs),
+        bar_format="{l_bar}{bar}| Time Left: {remaining}",
+    ):
         try:
-            task_status.update_state(state='PROGRESS', meta={'current': int((c1 / s1) * 100)})
-            store_add_texts_with_retry(store, i)
+            task_status.update_state(
+                state="PROGRESS", meta={"current": int((c1 / s1) * 100)}
+            )
+            store_add_texts_with_retry(store, i, id)
         except Exception as e:
             print(e)
             print("Error on ", i)
@@ -70,25 +73,3 @@ def call_openai_api(docs, folder_name, task_status):
         c1 += 1
     if settings.VECTOR_STORE == "faiss":
         store.save_local(f"{folder_name}")
-
-
-def get_user_permission(docs, folder_name):
-    # Function to ask user permission to call the OpenAI api and spend their OpenAI funds.
-    # Here we convert the docs list to a string and calculate the number of OpenAI tokens the string represents.
-    # docs_content = (" ".join(docs))
-    docs_content = ""
-    for doc in docs:
-        docs_content += doc.page_content
-
-    tokens, total_price = num_tokens_from_string(string=docs_content, encoding_name="cl100k_base")
-    # Here we print the number of tokens and the approx user cost with some visually appealing formatting.
-    print(f"Number of Tokens = {format(tokens, ',d')}")
-    print(f"Approx Cost = ${format(total_price, ',.2f')}")
-    # Here we check for user permission before calling the API.
-    user_input = input("Price Okay? (Y/N) \n").lower()
-    if user_input == "y":
-        call_openai_api(docs, folder_name)
-    elif user_input == "":
-        call_openai_api(docs, folder_name)
-    else:
-        print("The API was not called. No money was spent.")

application/parser/remote/base.py

@@ -0,0 +1,19 @@
+"""Base reader class."""
+from abc import abstractmethod
+from typing import Any, List
+
+from langchain.docstore.document import Document as LCDocument
+from application.parser.schema.base import Document
+
+
+class BaseRemote:
+    """Utilities for loading data from a directory."""
+
+    @abstractmethod
+    def load_data(self, *args: Any, **load_kwargs: Any) -> List[Document]:
+        """Load data from the input directory."""
+
+    def load_langchain_documents(self, **load_kwargs: Any) -> List[LCDocument]:
+        """Load data in LangChain document format."""
+        docs = self.load_data(**load_kwargs)
+        return [d.to_langchain_format() for d in docs]

application/parser/remote/crawler_loader.py

@@ -0,0 +1,59 @@
+import requests
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup
+from application.parser.remote.base import BaseRemote
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10):
+        from langchain_community.document_loaders import WebBaseLoader
+        self.loader = WebBaseLoader  # Initialize the document loader
+        self.limit = limit  # Set the limit for the number of pages to scrape
+
+    def load_data(self, inputs):
+        url = inputs
+        # Check if the input is a list and if it is, use the first element
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Check if the URL scheme is provided, if not, assume http
+        if not urlparse(url).scheme:
+            url = "http://" + url
+
+        visited_urls = set()  # Keep track of URLs that have been visited
+        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname  # Extract the base URL
+        urls_to_visit = [url]  # List of URLs to be visited, starting with the initial URL
+        loaded_content = []  # Store the loaded content from each URL
+
+        # Continue crawling until there are no more URLs to visit
+        while urls_to_visit:
+            current_url = urls_to_visit.pop(0)  # Get the next URL to visit
+            visited_urls.add(current_url)  # Mark the URL as visited
+
+            # Try to load and process the content from the current URL
+            try:
+                response = requests.get(current_url)  # Fetch the content of the current URL
+                response.raise_for_status()  # Raise an exception for HTTP errors
+                loader = self.loader([current_url])  # Initialize the document loader for the current URL
+                loaded_content.extend(loader.load())  # Load the content and add it to the loaded_content list
+            except Exception as e:
+                # Print an error message if loading or processing fails and continue with the next URL
+                print(f"Error processing URL {current_url}: {e}")
+                continue
+
+            # Parse the HTML content to extract all links
+            soup = BeautifulSoup(response.text, 'html.parser')
+            all_links = [
+                urljoin(current_url, a['href'])
+                for a in soup.find_all('a', href=True)
+                if base_url in urljoin(current_url, a['href'])  # Ensure links are from the same domain
+            ]
+
+            # Add new links to the list of URLs to visit if they haven't been visited yet
+            urls_to_visit.extend([link for link in all_links if link not in visited_urls])
+            urls_to_visit = list(set(urls_to_visit))  # Remove duplicate URLs
+
+            # Stop crawling if the limit of pages to scrape is reached
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return loaded_content  # Return the loaded content from all visited URLs

application/parser/remote/github_loader.py

@@ -0,0 +1,58 @@
+import base64
+import requests
+from typing import List
+from application.parser.remote.base import BaseRemote
+from langchain_core.documents import Document
+import mimetypes
+
+class GitHubLoader(BaseRemote):
+    def __init__(self):
+        self.access_token = None
+        self.headers = {
+            "Authorization": f"token {self.access_token}"
+        } if self.access_token else {}
+        return
+
+    def fetch_file_content(self, repo_url: str, file_path: str) -> str:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
+        response = requests.get(url, headers=self.headers)
+
+        if response.status_code == 200:
+            content = response.json()
+            mime_type, _ = mimetypes.guess_type(file_path)  # Guess the MIME type based on the file extension
+
+            if content.get("encoding") == "base64":
+                if mime_type and mime_type.startswith("text"):  # Handle only text files
+                    try:
+                        decoded_content = base64.b64decode(content["content"]).decode("utf-8")
+                        return f"Filename: {file_path}\n\n{decoded_content}"
+                    except Exception as e:
+                        raise e
+                else:
+                    return f"Filename: {file_path} is a binary file and was skipped."
+            else:
+                return f"Filename: {file_path}\n\n{content['content']}"
+        else:
+            response.raise_for_status()
+
+    def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
+        response = requests.get(url, headers={**self.headers, "Accept": "application/vnd.github.v3.raw"})
+        contents = response.json()
+        files = []
+        for item in contents:
+            if item["type"] == "file":
+                files.append(item["path"])
+            elif item["type"] == "dir":
+                files.extend(self.fetch_repo_files(repo_url, item["path"]))
+        return files
+
+    def load_data(self, repo_url: str) -> List[Document]:
+        repo_name = repo_url.split("github.com/")[-1]
+        files = self.fetch_repo_files(repo_name)
+        documents = []
+        for file_path in files:
+            content = self.fetch_file_content(repo_name, file_path)
+            documents.append(Document(page_content=content, metadata={"title": file_path, 
+            "source": f"https://github.com/{repo_name}/blob/main/{file_path}"}))
+        return documents

application/parser/remote/reddit_loader.py

@@ -0,0 +1,26 @@
+from application.parser.remote.base import BaseRemote
+from langchain_community.document_loaders import RedditPostsLoader
+
+
+class RedditPostsLoaderRemote(BaseRemote):
+    def load_data(self, inputs):
+        data = eval(inputs)
+        client_id = data.get("client_id")
+        client_secret = data.get("client_secret")
+        user_agent = data.get("user_agent")
+        categories = data.get("categories", ["new", "hot"])
+        mode = data.get("mode", "subreddit")
+        search_queries = data.get("search_queries")
+        number_posts = data.get("number_posts", 10)
+        self.loader = RedditPostsLoader(
+            client_id=client_id,
+            client_secret=client_secret,
+            user_agent=user_agent,
+            categories=categories,
+            mode=mode,
+            search_queries=search_queries,
+            number_posts=number_posts,
+        )
+        documents = self.loader.load()
+        print(f"Loaded {len(documents)} documents from Reddit")
+        return documents

application/parser/remote/remote_creator.py

@@ -0,0 +1,22 @@
+from application.parser.remote.sitemap_loader import SitemapLoader
+from application.parser.remote.crawler_loader import CrawlerLoader
+from application.parser.remote.web_loader import WebLoader
+from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
+from application.parser.remote.github_loader import GitHubLoader
+
+
+class RemoteCreator:
+    loaders = {
+        "url": WebLoader,
+        "sitemap": SitemapLoader,
+        "crawler": CrawlerLoader,
+        "reddit": RedditPostsLoaderRemote,
+        "github": GitHubLoader,
+    }
+
+    @classmethod
+    def create_loader(cls, type, *args, **kwargs):
+        loader_class = cls.loaders.get(type.lower())
+        if not loader_class:
+            raise ValueError(f"No LLM class found for type {type}")
+        return loader_class(*args, **kwargs)

application/parser/remote/sitemap_loader.py

@@ -0,0 +1,81 @@
+import requests
+import re  # Import regular expression library
+import xml.etree.ElementTree as ET
+from application.parser.remote.base import BaseRemote
+
+class SitemapLoader(BaseRemote):
+    def __init__(self, limit=20):
+        from langchain_community.document_loaders import WebBaseLoader
+        self.loader = WebBaseLoader
+        self.limit = limit  # Adding limit to control the number of URLs to process
+
+    def load_data(self, inputs):
+        sitemap_url= inputs
+        # Check if the input is a list and if it is, use the first element
+        if isinstance(sitemap_url, list) and sitemap_url:
+            url = sitemap_url[0]
+
+        urls = self._extract_urls(sitemap_url)
+        if not urls:
+            print(f"No URLs found in the sitemap: {sitemap_url}")
+            return []
+
+        # Load content of extracted URLs
+        documents = []
+        processed_urls = 0  # Counter for processed URLs
+        for url in urls:
+            if self.limit is not None and processed_urls >= self.limit:
+                break  # Stop processing if the limit is reached
+
+            try:
+                loader = self.loader([url])
+                documents.extend(loader.load())
+                processed_urls += 1  # Increment the counter after processing each URL
+            except Exception as e:
+                print(f"Error processing URL {url}: {e}")
+                continue
+
+        return documents
+
+    def _extract_urls(self, sitemap_url):
+        try:
+            response = requests.get(sitemap_url)
+            response.raise_for_status()  # Raise an exception for HTTP errors
+        except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
+            print(f"Failed to fetch sitemap: {sitemap_url}. Error: {e}")
+            return []
+
+        # Determine if this is a sitemap or a URL
+        if self._is_sitemap(response):
+            # It's a sitemap, so parse it and extract URLs
+            return self._parse_sitemap(response.content)
+        else:
+            # It's not a sitemap, return the URL itself
+            return [sitemap_url]
+
+    def _is_sitemap(self, response):
+        content_type = response.headers.get('Content-Type', '')
+        if 'xml' in content_type or response.url.endswith('.xml'):
+            return True
+
+        if '<sitemapindex' in response.text or '<urlset' in response.text:
+            return True
+
+        return False
+
+    def _parse_sitemap(self, sitemap_content):
+        # Remove namespaces
+        sitemap_content = re.sub(' xmlns="[^"]+"', '', sitemap_content.decode('utf-8'), count=1)
+
+        root = ET.fromstring(sitemap_content)
+
+        urls = []
+        for loc in root.findall('.//url/loc'):
+            urls.append(loc.text)
+
+        # Check for nested sitemaps
+        for sitemap in root.findall('.//sitemap/loc'):
+            nested_sitemap_url = sitemap.text
+            urls.extend(self._extract_urls(nested_sitemap_url))
+
+        return urls

application/parser/remote/telegram.py

@@ -0,0 +1,11 @@
+from langchain.document_loader import TelegramChatApiLoader
+from application.parser.remote.base import BaseRemote
+
+class TelegramChatApiRemote(BaseRemote):
+    def _init_parser(self, *args, **load_kwargs):
+        self.loader = TelegramChatApiLoader(**load_kwargs)
+        return {}
+
+    def parse_file(self, *args, **load_kwargs):
+
+        return 

application/parser/remote/web_loader.py

@@ -0,0 +1,32 @@
+from application.parser.remote.base import BaseRemote
+from langchain_community.document_loaders import WebBaseLoader
+
+headers = {
+    "User-Agent": "Mozilla/5.0",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*"
+    ";q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Referer": "https://www.google.com/",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+class WebLoader(BaseRemote):
+    def __init__(self):
+        self.loader = WebBaseLoader
+
+    def load_data(self, inputs):
+        urls = inputs
+        if isinstance(urls, str):
+            urls = [urls]
+        documents = []
+        for url in urls:
+            try:
+                loader = self.loader([url], header_template=headers)
+                documents.extend(loader.load())
+            except Exception as e:
+                print(f"Error processing URL {url}: {e}")
+                continue
+        return documents

application/parser/token_func.py

@@ -21,16 +21,18 @@ def group_documents(documents: List[Document], min_tokens: int, max_tokens: int)
     for doc in documents:
         doc_len = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
 
-        if current_group is None:
-            current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
-                                     extra_info=doc.extra_info)
-        elif len(tiktoken.get_encoding("cl100k_base").encode(
-                current_group.text)) + doc_len < max_tokens and doc_len < min_tokens:
-            current_group.text += " " + doc.text
+        # Check if current group is empty or if the document can be added based on token count and matching metadata
+        if (current_group is None or 
+            (len(tiktoken.get_encoding("cl100k_base").encode(current_group.text)) + doc_len < max_tokens and 
+             doc_len < min_tokens and 
+             current_group.extra_info == doc.extra_info)):
+            if current_group is None:
+                current_group = doc  # Use the document directly to retain its metadata
+            else:
+                current_group.text += " " + doc.text  # Append text to the current group
         else:
             docs.append(current_group)
-            current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
-                                     extra_info=doc.extra_info)
+            current_group = doc  # Start a new group with the current document
 
     if current_group is not None:
         docs.append(current_group)

application/requirements.txt

@@ -1,110 +1,89 @@
-aiodns==3.0.0
-aiohttp==3.8.6
-aiohttp-retry==2.8.3
-aiosignal==1.3.1
-aleph-alpha-client==2.16.1
-amqp==5.1.1
-anthropic==0.5.0
-async-timeout==4.0.2
-attrs==22.2.0
-billiard==3.6.4.0
-blobfile==2.0.1
-boto3==1.28.20
-celery==5.2.7
-cffi==1.15.1
-charset-normalizer==3.1.0
-click==8.1.3
-click-didyoumean==0.3.0
-click-plugins==1.1.1
-click-repl==0.2.0
-cryptography==41.0.4
-dataclasses-json==0.5.7
-decorator==5.1.1
+anthropic==0.34.2
+boto3==1.34.153
+beautifulsoup4==4.12.3
+celery==5.3.6
+dataclasses-json==0.6.7
 docx2txt==0.8
-dill==0.3.6
-dnspython==2.3.0
-ecdsa==0.18.0
-elasticsearch==8.9.0
-entrypoints==0.4
-faiss-cpu==1.7.3
-filelock==3.9.0
-Flask==2.2.5
-Flask-Cors==3.0.10
-frozenlist==1.3.3
-geojson==2.5.0
-gunicorn==20.1.0
-greenlet==2.0.2
-gpt4all==0.1.7
-huggingface-hub==0.15.1
-humbug==0.3.2
-idna==3.4
-itsdangerous==2.1.2
-Jinja2==3.1.2
+duckduckgo-search==6.3.0
+ebooklib==0.18
+elastic-transport==8.15.0
+elasticsearch==8.15.1
+escodegen==1.0.11
+esprima==4.0.1
+esutils==1.0.1
+Flask==3.0.3
+faiss-cpu==1.8.0.post1
+flask-restx==1.3.0
+gTTS==2.3.2
+gunicorn==23.0.0
+html2text==2024.2.26
+javalang==0.13.0
+jinja2==3.1.4
+jiter==0.5.0
 jmespath==1.0.1
-joblib==1.2.0
-kombu==5.2.4
-langchain==0.0.312
-loguru==0.6.0
-lxml==4.9.2
-MarkupSafe==2.1.2
-marshmallow==3.19.0
-marshmallow-enum==1.5.1
+joblib==1.4.2
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-spec==0.2.4
+jsonschema-specifications==2023.7.1
+kombu==5.4.2
+langchain==0.3.0
+langchain-community==0.3.0
+langchain-core==0.3.2
+langchain-openai==0.2.0
+langchain-text-splitters==0.3.0
+langsmith==0.1.125
+lazy-object-proxy==1.10.0
+lxml==5.3.0
+markupsafe==2.1.5
+marshmallow==3.22.0
 mpmath==1.3.0
-multidict==6.0.4
-multiprocess==0.70.14
+multidict==6.1.0
 mypy-extensions==1.0.0
-networkx==3.0
-npx
-nltk==3.8.1
-numcodecs==0.11.0
-numpy==1.24.2
-openai==0.27.8
-openapi3-parser==1.1.14
-packaging==23.0
-pathos==0.3.0
-Pillow==10.0.1
-pox==0.3.2
-ppft==1.7.6.6
-prompt-toolkit==3.0.38
+networkx==3.3
+numpy==1.26.4
+openai==1.46.1
+openapi-schema-validator==0.6.2
+openapi-spec-validator==0.6.0
+openapi3-parser==1.1.18
+orjson==3.10.7
+packaging==24.1
+pandas==2.2.3
+openpyxl==3.1.5
+pathable==0.4.3
+pillow==10.4.0
+portalocker==2.10.1
+prance==23.6.21.0
+primp==0.6.3
+prompt-toolkit==3.0.47
+protobuf==5.28.2
 py==1.11.0
-pyasn1==0.4.8
-pycares==4.3.0
-pycparser==2.21
-pycryptodomex==3.17
-pycryptodome==3.19.0
-pydantic==1.10.5
-PyJWT==2.6.0
-pymongo==4.3.3
-pyowm==3.3.0
-PyPDF2==3.0.1
-PySocks==1.7.1
-pytest
-python-dateutil==2.8.2
-python-dotenv==1.0.0
-python-jose==3.3.0
-pytz==2022.7.1
-PyYAML==6.0
-redis==4.5.4
-regex==2022.10.31
-requests==2.31.0
+pydantic==2.9.2
+pydantic-core==2.23.4
+pydantic-settings==2.4.0
+pymongo==4.8.0
+pypdf2==3.0.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-pptx==1.0.2
+qdrant-client==1.11.0
+redis==5.0.1
+referencing==0.30.2
+regex==2024.9.11
+requests==2.32.3
 retry==0.9.2
-rsa==4.9
-scikit-learn==1.2.2
-scipy==1.10.1
-sentencepiece
-six==1.16.0
-SQLAlchemy==1.4.46
-sympy==1.11.1
-tenacity==8.2.2
-threadpoolctl==3.1.0
-tiktoken
-tqdm==4.65.0
-transformers==4.30.0
-typer==0.7.0
-typing-inspect==0.8.0
-typing_extensions==4.5.0
-urllib3==1.26.18
-vine==5.0.0
-wcwidth==0.2.6
-yarl==1.8.2
-sentence-transformers==2.2.2
+sentence-transformers==3.0.1
+tiktoken==0.7.0
+tokenizers==0.19.1
+torch==2.4.1
+tqdm==4.66.5
+transformers==4.44.2
+typing-extensions==4.12.2
+typing-inspect==0.9.0
+tzdata==2024.2
+urllib3==2.2.3
+vine==5.1.0
+wcwidth==0.2.13
+werkzeug==3.0.4
+yarl==1.11.1

application/retriever/base.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+
+
+class BaseRetriever(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def gen(self, *args, **kwargs):
+        pass
+
+    @abstractmethod
+    def search(self, *args, **kwargs):
+        pass
+
+    @abstractmethod
+    def get_params(self):
+        pass

application/retriever/brave_search.py

@@ -0,0 +1,114 @@
+import json
+from application.retriever.base import BaseRetriever
+from application.core.settings import settings
+from application.llm.llm_creator import LLMCreator
+from application.utils import num_tokens_from_string
+from langchain_community.tools import BraveSearch
+
+
+class BraveRetSearch(BaseRetriever):
+
+    def __init__(
+        self,
+        question,
+        source,
+        chat_history,
+        prompt,
+        chunks=2,
+        token_limit=150,
+        gpt_model="docsgpt",
+        user_api_key=None,
+    ):
+        self.question = question
+        self.source = source
+        self.chat_history = chat_history
+        self.prompt = prompt
+        self.chunks = chunks
+        self.gpt_model = gpt_model
+        self.token_limit = (
+            token_limit
+            if token_limit
+            < settings.MODEL_TOKEN_LIMITS.get(
+                self.gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+            else settings.MODEL_TOKEN_LIMITS.get(
+                self.gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+        )
+        self.user_api_key = user_api_key
+
+    def _get_data(self):
+        if self.chunks == 0:
+            docs = []
+        else:
+            search = BraveSearch.from_api_key(
+                api_key=settings.BRAVE_SEARCH_API_KEY,
+                search_kwargs={"count": int(self.chunks)},
+            )
+            results = search.run(self.question)
+            results = json.loads(results)
+
+            docs = []
+            for i in results:
+                try:
+                    title = i["title"]
+                    link = i["link"]
+                    snippet = i["snippet"]
+                    docs.append({"text": snippet, "title": title, "link": link})
+                except IndexError:
+                    pass
+        if settings.LLM_NAME == "llama.cpp":
+            docs = [docs[0]]
+
+        return docs
+
+    def gen(self):
+        docs = self._get_data()
+
+        # join all page_content together with a newline
+        docs_together = "\n".join([doc["text"] for doc in docs])
+        p_chat_combine = self.prompt.replace("{summaries}", docs_together)
+        messages_combine = [{"role": "system", "content": p_chat_combine}]
+        for doc in docs:
+            yield {"source": doc}
+
+        if len(self.chat_history) > 1:
+            tokens_current_history = 0
+            # count tokens in history
+            for i in self.chat_history:
+                if "prompt" in i and "response" in i:
+                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
+                        i["response"]
+                    )
+                    if tokens_current_history + tokens_batch < self.token_limit:
+                        tokens_current_history += tokens_batch
+                        messages_combine.append(
+                            {"role": "user", "content": i["prompt"]}
+                        )
+                        messages_combine.append(
+                            {"role": "system", "content": i["response"]}
+                        )
+        messages_combine.append({"role": "user", "content": self.question})
+
+        llm = LLMCreator.create_llm(
+            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        )
+
+        completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        for line in completion:
+            yield {"answer": str(line)}
+
+    def search(self):
+        return self._get_data()
+
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/classic_rag.py

@@ -0,0 +1,116 @@
+from application.retriever.base import BaseRetriever
+from application.core.settings import settings
+from application.vectorstore.vector_creator import VectorCreator
+from application.llm.llm_creator import LLMCreator
+
+from application.utils import num_tokens_from_string
+
+
+class ClassicRAG(BaseRetriever):
+
+    def __init__(
+        self,
+        question,
+        source,
+        chat_history,
+        prompt,
+        chunks=2,
+        token_limit=150,
+        gpt_model="docsgpt",
+        user_api_key=None,
+    ):
+        self.question = question
+        self.vectorstore = source['active_docs'] if 'active_docs' in source else None
+        self.chat_history = chat_history
+        self.prompt = prompt
+        self.chunks = chunks
+        self.gpt_model = gpt_model
+        self.token_limit = (
+            token_limit
+            if token_limit
+            < settings.MODEL_TOKEN_LIMITS.get(
+                self.gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+            else settings.MODEL_TOKEN_LIMITS.get(
+                self.gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+        )
+        self.user_api_key = user_api_key
+
+    def _get_data(self):
+        if self.chunks == 0:
+            docs = []
+        else:
+            docsearch = VectorCreator.create_vectorstore(
+                settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
+            )
+            docs_temp = docsearch.search(self.question, k=self.chunks)
+            print(docs_temp)
+            docs = [
+                {
+                    "title": i.metadata.get(
+                        "title", i.metadata.get("post_title", i.page_content)
+                    ).split("/")[-1],
+                    "text": i.page_content,
+                    "source": (
+                        i.metadata.get("source")
+                        if i.metadata.get("source")
+                        else "local"
+                    ),
+                }
+                for i in docs_temp
+            ]
+        if settings.LLM_NAME == "llama.cpp":
+            docs = [docs[0]]
+
+        return docs
+
+    def gen(self):
+        docs = self._get_data()
+
+        # join all page_content together with a newline
+        docs_together = "\n".join([doc["text"] for doc in docs])
+        p_chat_combine = self.prompt.replace("{summaries}", docs_together)
+        messages_combine = [{"role": "system", "content": p_chat_combine}]
+        for doc in docs:
+            yield {"source": doc}
+
+        if len(self.chat_history) > 1:
+            tokens_current_history = 0
+            # count tokens in history
+            for i in self.chat_history:
+                if "prompt" in i and "response" in i:
+                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
+                        i["response"]
+                    )
+                    if tokens_current_history + tokens_batch < self.token_limit:
+                        tokens_current_history += tokens_batch
+                        messages_combine.append(
+                            {"role": "user", "content": i["prompt"]}
+                        )
+                        messages_combine.append(
+                            {"role": "system", "content": i["response"]}
+                        )
+        messages_combine.append({"role": "user", "content": self.question})
+
+        llm = LLMCreator.create_llm(
+            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        )
+        completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        for line in completion:
+            yield {"answer": str(line)}
+
+    def search(self):
+        return self._get_data()
+    
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.vectorstore,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/duckduck_search.py

@@ -0,0 +1,131 @@
+from application.retriever.base import BaseRetriever
+from application.core.settings import settings
+from application.llm.llm_creator import LLMCreator
+from application.utils import num_tokens_from_string
+from langchain_community.tools import DuckDuckGoSearchResults
+from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
+
+
+class DuckDuckSearch(BaseRetriever):
+
+    def __init__(
+        self,
+        question,
+        source,
+        chat_history,
+        prompt,
+        chunks=2,
+        token_limit=150,
+        gpt_model="docsgpt",
+        user_api_key=None,
+    ):
+        self.question = question
+        self.source = source
+        self.chat_history = chat_history
+        self.prompt = prompt
+        self.chunks = chunks
+        self.gpt_model = gpt_model
+        self.token_limit = (
+            token_limit
+            if token_limit
+            < settings.MODEL_TOKEN_LIMITS.get(
+                self.gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+            else settings.MODEL_TOKEN_LIMITS.get(
+                self.gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+        )
+        self.user_api_key = user_api_key
+
+    def _parse_lang_string(self, input_string):
+        result = []
+        current_item = ""
+        inside_brackets = False
+        for char in input_string:
+            if char == "[":
+                inside_brackets = True
+            elif char == "]":
+                inside_brackets = False
+                result.append(current_item)
+                current_item = ""
+            elif inside_brackets:
+                current_item += char
+
+        if inside_brackets:
+            result.append(current_item)
+
+        return result
+
+    def _get_data(self):
+        if self.chunks == 0:
+            docs = []
+        else:
+            wrapper = DuckDuckGoSearchAPIWrapper(max_results=self.chunks)
+            search = DuckDuckGoSearchResults(api_wrapper=wrapper)
+            results = search.run(self.question)
+            results = self._parse_lang_string(results)
+
+            docs = []
+            for i in results:
+                try:
+                    text = i.split("title:")[0]
+                    title = i.split("title:")[1].split("link:")[0]
+                    link = i.split("link:")[1]
+                    docs.append({"text": text, "title": title, "link": link})
+                except IndexError:
+                    pass
+        if settings.LLM_NAME == "llama.cpp":
+            docs = [docs[0]]
+
+        return docs
+
+    def gen(self):
+        docs = self._get_data()
+
+        # join all page_content together with a newline
+        docs_together = "\n".join([doc["text"] for doc in docs])
+        p_chat_combine = self.prompt.replace("{summaries}", docs_together)
+        messages_combine = [{"role": "system", "content": p_chat_combine}]
+        for doc in docs:
+            yield {"source": doc}
+
+        if len(self.chat_history) > 1:
+            tokens_current_history = 0
+            # count tokens in history
+            for i in self.chat_history:
+                if "prompt" in i and "response" in i:
+                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
+                        i["response"]
+                    )
+                    if tokens_current_history + tokens_batch < self.token_limit:
+                        tokens_current_history += tokens_batch
+                        messages_combine.append(
+                            {"role": "user", "content": i["prompt"]}
+                        )
+                        messages_combine.append(
+                            {"role": "system", "content": i["response"]}
+                        )
+        messages_combine.append({"role": "user", "content": self.question})
+
+        llm = LLMCreator.create_llm(
+            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        )
+
+        completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        for line in completion:
+            yield {"answer": str(line)}
+
+    def search(self):
+        return self._get_data()
+    
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/retriever_creator.py

@@ -0,0 +1,20 @@
+from application.retriever.classic_rag import ClassicRAG
+from application.retriever.duckduck_search import DuckDuckSearch
+from application.retriever.brave_search import BraveRetSearch
+
+
+
+class RetrieverCreator:
+    retrievers = {
+        'classic': ClassicRAG,
+        'duckduck_search': DuckDuckSearch,
+        'brave_search': BraveRetSearch,
+        'default': ClassicRAG
+    }
+
+    @classmethod
+    def create_retriever(cls, type, *args, **kwargs):
+        retiever_class = cls.retrievers.get(type.lower())
+        if not retiever_class:
+            raise ValueError(f"No retievers class found for type {type}")
+        return retiever_class(*args, **kwargs)

application/tts/base.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+
+
+class BaseTTS(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def text_to_speech(self, *args, **kwargs):
+        pass

application/tts/elevenlabs.py

@@ -0,0 +1,29 @@
+from io import BytesIO
+import base64
+from application.tts.base import BaseTTS
+
+
+class ElevenlabsTTS(BaseTTS):
+    def __init__(self):
+        from elevenlabs.client import ElevenLabs
+
+        self.client = ElevenLabs(
+            api_key="ELEVENLABS_API_KEY",
+            )
+    
+
+    def text_to_speech(self, text):
+        lang = "en"
+        audio = self.client.generate(
+            text=text,
+            model="eleven_multilingual_v2",
+            voice="Brian",
+        )
+        audio_data = BytesIO()
+        for chunk in audio:
+            audio_data.write(chunk)
+        audio_bytes = audio_data.getvalue()
+
+        # Encode to base64
+        audio_base64 = base64.b64encode(audio_bytes).decode("utf-8")
+        return audio_base64, lang

application/tts/google_tts.py

@@ -0,0 +1,19 @@
+import io
+import base64
+from gtts import gTTS
+from application.tts.base import BaseTTS
+
+
+class GoogleTTS(BaseTTS):
+    def __init__(self):
+        pass
+
+
+    def text_to_speech(self, text):
+        lang = "en"
+        audio_fp = io.BytesIO()
+        tts = gTTS(text=text, lang=lang, slow=False)
+        tts.write_to_fp(audio_fp)
+        audio_fp.seek(0)
+        audio_base64 = base64.b64encode(audio_fp.read()).decode("utf-8")
+        return audio_base64, lang

application/usage.py

@@ -0,0 +1,48 @@
+import sys
+from datetime import datetime
+from application.core.mongo_db import MongoDB
+from application.utils import num_tokens_from_string
+
+mongo = MongoDB.get_client()
+db = mongo["docsgpt"]
+usage_collection = db["token_usage"]
+
+
+def update_token_usage(user_api_key, token_usage):
+    if "pytest" in sys.modules:
+        return
+    usage_data = {
+        "api_key": user_api_key,
+        "prompt_tokens": token_usage["prompt_tokens"],
+        "generated_tokens": token_usage["generated_tokens"],
+        "timestamp": datetime.now(),
+    }
+    usage_collection.insert_one(usage_data)
+
+
+def gen_token_usage(func):
+    def wrapper(self, model, messages, stream, **kwargs):
+        for message in messages:
+            self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
+        result = func(self, model, messages, stream, **kwargs)
+        self.token_usage["generated_tokens"] += num_tokens_from_string(result)
+        update_token_usage(self.user_api_key, self.token_usage)
+        return result
+
+    return wrapper
+
+
+def stream_token_usage(func):
+    def wrapper(self, model, messages, stream, **kwargs):
+        for message in messages:
+            self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
+        batch = []
+        result = func(self, model, messages, stream, **kwargs)
+        for r in result:
+            batch.append(r)
+            yield r
+        for line in batch:
+            self.token_usage["generated_tokens"] += num_tokens_from_string(line)
+        update_token_usage(self.user_api_key, self.token_usage)
+
+    return wrapper

application/utils.py

@@ -0,0 +1,48 @@
+import tiktoken
+import hashlib
+from flask import jsonify, make_response
+
+
+_encoding = None
+
+
+def get_encoding():
+    global _encoding
+    if _encoding is None:
+        _encoding = tiktoken.get_encoding("cl100k_base")
+    return _encoding
+
+
+def num_tokens_from_string(string: str) -> int:
+    encoding = get_encoding()
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
+
+
+def count_tokens_docs(docs):
+    docs_content = ""
+    for doc in docs:
+        docs_content += doc.page_content
+
+    tokens = num_tokens_from_string(docs_content)
+    return tokens
+
+
+def check_required_fields(data, required_fields):
+    missing_fields = [field for field in required_fields if field not in data]
+    if missing_fields:
+        return make_response(
+            jsonify(
+                {
+                    "success": False,
+                    "message": f"Missing fields: {', '.join(missing_fields)}",
+                }
+            ),
+            400,
+        )
+    return None
+
+
+def get_hash(data):
+    return hashlib.md5(data.encode()).hexdigest()
+

application/vectorstore/base.py

@@ -1,13 +1,55 @@
 from abc import ABC, abstractmethod
 import os
-from langchain.embeddings import (
-    OpenAIEmbeddings,
-    HuggingFaceEmbeddings,
-    CohereEmbeddings,
-    HuggingFaceInstructEmbeddings,
-)
+from sentence_transformers import SentenceTransformer
+from langchain_openai import OpenAIEmbeddings
 from application.core.settings import settings
 
+class EmbeddingsWrapper:
+    def __init__(self, model_name, *args, **kwargs):
+        self.model = SentenceTransformer(model_name, config_kwargs={'allow_dangerous_deserialization': True}, *args, **kwargs)
+        self.dimension = self.model.get_sentence_embedding_dimension()
+
+    def embed_query(self, query: str):
+        return self.model.encode(query).tolist()
+    
+    def embed_documents(self, documents: list):
+        return self.model.encode(documents).tolist()
+    
+    def __call__(self, text):
+        if isinstance(text, str):
+            return self.embed_query(text)
+        elif isinstance(text, list):
+            return self.embed_documents(text)
+        else:
+            raise ValueError("Input must be a string or a list of strings")
+
+
+
+class EmbeddingsSingleton:
+    _instances = {}
+
+    @staticmethod
+    def get_instance(embeddings_name, *args, **kwargs):
+        if embeddings_name not in EmbeddingsSingleton._instances:
+            EmbeddingsSingleton._instances[embeddings_name] = EmbeddingsSingleton._create_instance(
+                embeddings_name, *args, **kwargs
+            )
+        return EmbeddingsSingleton._instances[embeddings_name]
+
+    @staticmethod
+    def _create_instance(embeddings_name, *args, **kwargs):
+        embeddings_factory = {
+            "openai_text-embedding-ada-002": OpenAIEmbeddings,
+            "huggingface_sentence-transformers/all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
+            "huggingface_sentence-transformers-all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
+            "huggingface_hkunlp/instructor-large": lambda: EmbeddingsWrapper("hkunlp/instructor-large"),
+        }
+
+        if embeddings_name in embeddings_factory:
+            return embeddings_factory[embeddings_name](*args, **kwargs)
+        else:
+            return EmbeddingsWrapper(embeddings_name, *args, **kwargs)
+
 class BaseVectorStore(ABC):
     def __init__(self):
         pass
@@ -20,32 +62,28 @@ class BaseVectorStore(ABC):
         return settings.OPENAI_API_BASE and settings.OPENAI_API_VERSION and settings.AZURE_DEPLOYMENT_NAME
 
     def _get_embeddings(self, embeddings_name, embeddings_key=None):
-        embeddings_factory = {
-            "openai_text-embedding-ada-002": OpenAIEmbeddings,
-            "huggingface_sentence-transformers/all-mpnet-base-v2": HuggingFaceEmbeddings,
-            "huggingface_hkunlp/instructor-large": HuggingFaceInstructEmbeddings,
-            "cohere_medium": CohereEmbeddings
-        }
-        
-        if embeddings_name not in embeddings_factory:
-            raise ValueError(f"Invalid embeddings_name: {embeddings_name}")
-
         if embeddings_name == "openai_text-embedding-ada-002":
             if self.is_azure_configured():
                 os.environ["OPENAI_API_TYPE"] = "azure"
-                embedding_instance = embeddings_factory[embeddings_name](
+                embedding_instance = EmbeddingsSingleton.get_instance(
+                    embeddings_name,
                     model=settings.AZURE_EMBEDDINGS_DEPLOYMENT_NAME
                 )
             else:
-                embedding_instance = embeddings_factory[embeddings_name](
+                embedding_instance = EmbeddingsSingleton.get_instance(
+                    embeddings_name,
                     openai_api_key=embeddings_key
                 )
-        elif embeddings_name == "cohere_medium":
-            embedding_instance = embeddings_factory[embeddings_name](
-                cohere_api_key=embeddings_key
-            )
+        elif embeddings_name == "huggingface_sentence-transformers/all-mpnet-base-v2":
+            if os.path.exists("./model/all-mpnet-base-v2"):
+                embedding_instance = EmbeddingsSingleton.get_instance(
+                    embeddings_name="./model/all-mpnet-base-v2",
+                )
+            else:
+                embedding_instance = EmbeddingsSingleton.get_instance(
+                    embeddings_name,
+                )
         else:
-            embedding_instance = embeddings_factory[embeddings_name]()
-            
-        return embedding_instance
+            embedding_instance = EmbeddingsSingleton.get_instance(embeddings_name)
 
+        return embedding_instance

application/vectorstore/document_class.py

@@ -0,0 +1,8 @@
+class Document(str):
+    """Class for storing a piece of text and associated metadata."""
+
+    def __new__(cls, page_content: str, metadata: dict):
+        instance = super().__new__(cls, page_content)
+        instance.page_content = page_content
+        instance.metadata = metadata
+        return instance

application/vectorstore/elasticsearch.py

@@ -1,25 +1,17 @@
 from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
+from application.vectorstore.document_class import Document
 import elasticsearch
 
-class Document(str):
-    """Class for storing a piece of text and associated metadata."""
-
-    def __new__(cls, page_content: str, metadata: dict):
-        instance = super().__new__(cls, page_content)
-        instance.page_content = page_content
-        instance.metadata = metadata
-        return instance
-
 
 
 
 class ElasticsearchStore(BaseVectorStore):
     _es_connection = None  # Class attribute to hold the Elasticsearch connection
 
-    def __init__(self, path, embeddings_key, index_name=settings.ELASTIC_INDEX):
+    def __init__(self, source_id, embeddings_key, index_name=settings.ELASTIC_INDEX):
         super().__init__()
-        self.path = path.replace("application/indexes/", "").rstrip("/")
+        self.source_id = source_id.replace("application/indexes/", "").rstrip("/")
         self.embeddings_key = embeddings_key
         self.index_name = index_name
         
@@ -89,7 +81,7 @@ class ElasticsearchStore(BaseVectorStore):
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
         vector = embeddings.embed_query(question)
         knn = {
-            "filter": [{"match": {"metadata.store.keyword": self.path}}],
+            "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
             "field": "vector",
             "k": k,
             "num_candidates": 100,
@@ -108,7 +100,7 @@ class ElasticsearchStore(BaseVectorStore):
                             }
                         }
                     ],
-                    "filter": [{"match": {"metadata.store.keyword": self.path}}],
+                    "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
                 }
             },
             "rank": {"rrf": {}},
@@ -217,5 +209,4 @@ class ElasticsearchStore(BaseVectorStore):
 
     def delete_index(self):
         self._es_connection.delete_by_query(index=self.index_name, query={"match": {
-                                      "metadata.store.keyword": self.path}},)
-
+                                      "metadata.source_id.keyword": self.source_id}},)

application/vectorstore/faiss.py

@@ -1,21 +1,29 @@
-from langchain.vectorstores import FAISS
+from langchain_community.vectorstores import FAISS
 from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
+import os
+
+def get_vectorstore(path: str) -> str:
+    if path:
+        vectorstore = os.path.join("application", "indexes", path)
+    else:
+        vectorstore = os.path.join("application")
+    return vectorstore
 
 class FaissStore(BaseVectorStore):
-
-    def __init__(self, path, embeddings_key, docs_init=None):
+    def __init__(self, source_id: str, embeddings_key: str, docs_init=None):
         super().__init__()
-        self.path = path
+        self.path = get_vectorstore(source_id)
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
-        if docs_init:
-            self.docsearch = FAISS.from_documents(
-                docs_init, embeddings
-            )
-        else:
-            self.docsearch = FAISS.load_local(
-                self.path, embeddings
-            )
+
+        try:
+            if docs_init:
+                self.docsearch = FAISS.from_documents(docs_init, embeddings)
+            else:
+                self.docsearch = FAISS.load_local(self.path, embeddings, allow_dangerous_deserialization=True)
+        except Exception:
+            raise
+
         self.assert_embedding_dimensions(embeddings)
 
     def search(self, *args, **kwargs):
@@ -31,16 +39,12 @@ class FaissStore(BaseVectorStore):
         return self.docsearch.delete(*args, **kwargs)
 
     def assert_embedding_dimensions(self, embeddings):
-        """
-        Check that the word embedding dimension of the docsearch index matches
-        the dimension of the word embeddings used 
-        """
+        """Check that the word embedding dimension of the docsearch index matches the dimension of the word embeddings used."""
         if settings.EMBEDDINGS_NAME == "huggingface_sentence-transformers/all-mpnet-base-v2":
-            try:
-                word_embedding_dimension = embeddings.client[1].word_embedding_dimension
-            except AttributeError as e:
-                raise AttributeError("word_embedding_dimension not found in embeddings.client[1]") from e
+            word_embedding_dimension = getattr(embeddings, 'dimension', None)
+            if word_embedding_dimension is None:
+                raise AttributeError("'dimension' attribute not found in embeddings instance.")
+            
             docsearch_index_dimension = self.docsearch.index.d
             if word_embedding_dimension != docsearch_index_dimension:
-                raise ValueError(f"word_embedding_dimension ({word_embedding_dimension}) " +
-                                 f"!= docsearch_index_word_embedding_dimension ({docsearch_index_dimension})")
+                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) != docsearch index dimension ({docsearch_index_dimension})")

application/vectorstore/lancedb.py

@@ -0,0 +1,119 @@
+from typing import List, Optional
+import importlib
+from application.vectorstore.base import BaseVectorStore
+from application.core.settings import settings
+
+class LanceDBVectorStore(BaseVectorStore):
+    """Class for LanceDB Vector Store integration."""
+
+    def __init__(self, path: str = settings.LANCEDB_PATH,
+                 table_name_prefix: str = settings.LANCEDB_TABLE_NAME,
+                 source_id: str = None,
+                 embeddings_key: str = "embeddings"):
+        """Initialize the LanceDB vector store."""
+        super().__init__()
+        self.path = path
+        self.table_name = f"{table_name_prefix}_{source_id}" if source_id else table_name_prefix
+        self.embeddings_key = embeddings_key
+        self._lance_db = None
+        self.docsearch = None
+        self._pa = None  # PyArrow (pa) will be lazy loaded
+
+    @property
+    def pa(self):
+        """Lazy load pyarrow module."""
+        if self._pa is None:
+            self._pa = importlib.import_module("pyarrow")
+        return self._pa
+
+    @property
+    def lancedb(self):
+        """Lazy load lancedb module."""
+        if not hasattr(self, "_lancedb_module"):
+            self._lancedb_module = importlib.import_module("lancedb")
+        return self._lancedb_module
+
+    @property
+    def lance_db(self):
+        """Lazy load the LanceDB connection."""
+        if self._lance_db is None:
+            self._lance_db = self.lancedb.connect(self.path)
+        return self._lance_db
+
+    @property
+    def table(self):
+        """Lazy load the LanceDB table."""
+        if self.docsearch is None:
+            if self.table_name in self.lance_db.table_names():
+                self.docsearch = self.lance_db.open_table(self.table_name)
+            else:
+                self.docsearch = None
+        return self.docsearch
+
+    def ensure_table_exists(self):
+        """Ensure the table exists before performing operations."""
+        if self.table is None:
+            embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
+            schema = self.pa.schema([
+                self.pa.field("vector", self.pa.list_(self.pa.float32(), list_size=embeddings.dimension)),
+                self.pa.field("text", self.pa.string()),
+                self.pa.field("metadata", self.pa.struct([
+                    self.pa.field("key", self.pa.string()),
+                    self.pa.field("value", self.pa.string())
+                ]))
+            ])
+            self.docsearch = self.lance_db.create_table(self.table_name, schema=schema)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]] = None, source_id: str = None):
+        """Add texts with metadata and their embeddings to the LanceDB table."""
+        embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_documents(texts)
+        vectors = []
+        for embedding, text, metadata in zip(embeddings, texts, metadatas or [{}] * len(texts)):
+            if source_id:
+                metadata["source_id"] = source_id
+            metadata_struct = [{"key": k, "value": str(v)} for k, v in metadata.items()]
+            vectors.append({
+                "vector": embedding,
+                "text": text,
+                "metadata": metadata_struct
+            })
+        self.ensure_table_exists()
+        self.docsearch.add(vectors)
+
+    def search(self, query: str, k: int = 2, *args, **kwargs):
+        """Search LanceDB for the top k most similar vectors."""
+        self.ensure_table_exists()
+        query_embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_query(query)
+        results = self.docsearch.search(query_embedding).limit(k).to_list()
+        return [(result["_distance"], result["text"], result["metadata"]) for result in results]
+
+    def delete_index(self):
+        """Delete the entire LanceDB index (table)."""
+        if self.table:
+            self.lance_db.drop_table(self.table_name)
+
+    def assert_embedding_dimensions(self, embeddings):
+        """Ensure that embedding dimensions match the table index dimensions."""
+        word_embedding_dimension = embeddings.dimension
+        if self.table:
+            table_index_dimension = len(self.docsearch.schema["vector"].type.value_type)
+            if word_embedding_dimension != table_index_dimension:
+                raise ValueError(
+                    f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) "
+                    f"!= table index dimension ({table_index_dimension})"
+                )
+
+    def filter_documents(self, filter_condition: dict) -> List[dict]:
+        """Filter documents based on certain conditions."""
+        self.ensure_table_exists()
+
+        # Ensure source_id exists in the filter condition
+        if 'source_id' not in filter_condition:
+            raise ValueError("filter_condition must contain 'source_id'")
+
+        source_id = filter_condition["source_id"]
+
+        # Use LanceDB's native filtering if supported, otherwise filter manually
+        filtered_data = self.docsearch.filter(lambda x: x.metadata and x.metadata.get("source_id") == source_id).to_list()
+
+        return filtered_data

application/vectorstore/milvus.py

@@ -0,0 +1,38 @@
+from typing import List, Optional
+from uuid import uuid4
+
+
+from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
+
+
+class MilvusStore(BaseVectorStore):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
+        super().__init__()
+        from langchain_milvus import Milvus
+
+        connection_args = {
+            "uri": settings.MILVUS_URI,
+            "token": settings.MILVUS_TOKEN,
+        }
+        self._docsearch = Milvus(
+            embedding_function=self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key),
+            collection_name=settings.MILVUS_COLLECTION_NAME,
+            connection_args=connection_args,
+        )
+        self._source_id = source_id
+
+    def search(self, question, k=2, *args, **kwargs):
+        expr = f"source_id == '{self._source_id}'"
+        return self._docsearch.similarity_search(query=question, k=k, expr=expr, *args, **kwargs)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]], *args, **kwargs):
+        ids = [str(uuid4()) for _ in range(len(texts))]
+
+        return self._docsearch.add_texts(texts=texts, metadatas=metadatas, ids=ids, *args, **kwargs)
+
+    def save_local(self, *args, **kwargs):
+        pass
+
+    def delete_index(self, *args, **kwargs):
+        pass

application/vectorstore/mongodb.py

@@ -0,0 +1,126 @@
+from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
+from application.vectorstore.document_class import Document
+
+
+class MongoDBVectorStore(BaseVectorStore):
+    def __init__(
+        self,
+        source_id: str = "",
+        embeddings_key: str = "embeddings",
+        collection: str = "documents",
+        index_name: str = "vector_search_index",
+        text_key: str = "text",
+        embedding_key: str = "embedding",
+        database: str = "docsgpt",
+    ):
+        self._index_name = index_name
+        self._text_key = text_key
+        self._embedding_key = embedding_key
+        self._embeddings_key = embeddings_key
+        self._mongo_uri = settings.MONGO_URI
+        self._source_id = source_id.replace("application/indexes/", "").rstrip("/")
+        self._embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
+
+        try:
+            import pymongo
+        except ImportError:
+            raise ImportError(
+                "Could not import pymongo python package. "
+                "Please install it with `pip install pymongo`."
+            )
+
+        self._client = pymongo.MongoClient(self._mongo_uri)
+        self._database = self._client[database]
+        self._collection = self._database[collection]
+
+    def search(self, question, k=2, *args, **kwargs):
+        query_vector = self._embedding.embed_query(question)
+
+        pipeline = [
+            {
+                "$vectorSearch": {
+                    "queryVector": query_vector,
+                    "path": self._embedding_key,
+                    "limit": k,
+                    "numCandidates": k * 10,
+                    "index": self._index_name,
+                    "filter": {"source_id": {"$eq": self._source_id}},
+                }
+            }
+        ]
+
+        cursor = self._collection.aggregate(pipeline)
+
+        results = []
+        for doc in cursor:
+            text = doc[self._text_key]
+            doc.pop("_id")
+            doc.pop(self._text_key)
+            doc.pop(self._embedding_key)
+            metadata = doc
+            results.append(Document(text, metadata))
+        return results
+
+    def _insert_texts(self, texts, metadatas):
+        if not texts:
+            return []
+        embeddings = self._embedding.embed_documents(texts)
+
+        to_insert = [
+            {self._text_key: t, self._embedding_key: embedding, **m}
+            for t, m, embedding in zip(texts, metadatas, embeddings)
+        ]
+
+        insert_result = self._collection.insert_many(to_insert)
+        return insert_result.inserted_ids
+
+    def add_texts(
+        self,
+        texts,
+        metadatas=None,
+        ids=None,
+        refresh_indices=True,
+        create_index_if_not_exists=True,
+        bulk_kwargs=None,
+        **kwargs,
+    ):
+
+        # dims = self._embedding.client[1].word_embedding_dimension
+        # # check if index exists
+        # if create_index_if_not_exists:
+        #     # check if index exists
+        #     info = self._collection.index_information()
+        #     if self._index_name not in info:
+        #         index_mongo = {
+        #         "fields": [{
+        #             "type": "vector",
+        #             "path": self._embedding_key,
+        #             "numDimensions": dims,
+        #             "similarity": "cosine",
+        #         },
+        #         {
+        #             "type": "filter",
+        #             "path": "store"
+        #         }]
+        #         }
+        #         self._collection.create_index(self._index_name, index_mongo)
+
+        batch_size = 100
+        _metadatas = metadatas or ({} for _ in texts)
+        texts_batch = []
+        metadatas_batch = []
+        result_ids = []
+        for i, (text, metadata) in enumerate(zip(texts, _metadatas)):
+            texts_batch.append(text)
+            metadatas_batch.append(metadata)
+            if (i + 1) % batch_size == 0:
+                result_ids.extend(self._insert_texts(texts_batch, metadatas_batch))
+                texts_batch = []
+                metadatas_batch = []
+        if texts_batch:
+            result_ids.extend(self._insert_texts(texts_batch, metadatas_batch))
+        return result_ids
+
+    def delete_index(self, *args, **kwargs):
+        self._collection.delete_many({"source_id": self._source_id})

application/vectorstore/qdrant.py

@@ -0,0 +1,47 @@
+from langchain_community.vectorstores.qdrant import Qdrant
+from application.vectorstore.base import BaseVectorStore
+from application.core.settings import settings
+from qdrant_client import models
+
+
+class QdrantStore(BaseVectorStore):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
+        self._filter = models.Filter(
+            must=[
+                models.FieldCondition(
+                    key="metadata.source_id",
+                    match=models.MatchValue(value=source_id.replace("application/indexes/", "").rstrip("/")),
+                )
+            ]
+        )
+
+        self._docsearch = Qdrant.construct_instance(
+            ["TEXT_TO_OBTAIN_EMBEDDINGS_DIMENSION"],
+            embedding=self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key),
+            collection_name=settings.QDRANT_COLLECTION_NAME,
+            location=settings.QDRANT_LOCATION,
+            url=settings.QDRANT_URL,
+            port=settings.QDRANT_PORT,
+            grpc_port=settings.QDRANT_GRPC_PORT,
+            https=settings.QDRANT_HTTPS,
+            prefer_grpc=settings.QDRANT_PREFER_GRPC,
+            api_key=settings.QDRANT_API_KEY,
+            prefix=settings.QDRANT_PREFIX,
+            timeout=settings.QDRANT_TIMEOUT,
+            path=settings.QDRANT_PATH,
+            distance_func=settings.QDRANT_DISTANCE_FUNC,
+        )
+
+    def search(self, *args, **kwargs):
+        return self._docsearch.similarity_search(filter=self._filter, *args, **kwargs)
+
+    def add_texts(self, *args, **kwargs):
+        return self._docsearch.add_texts(*args, **kwargs)
+
+    def save_local(self, *args, **kwargs):
+        pass
+
+    def delete_index(self, *args, **kwargs):
+        return self._docsearch.client.delete(
+            collection_name=settings.QDRANT_COLLECTION_NAME, points_selector=self._filter
+        )

application/vectorstore/vector_creator.py

@@ -1,11 +1,17 @@
 from application.vectorstore.faiss import FaissStore
 from application.vectorstore.elasticsearch import ElasticsearchStore
+from application.vectorstore.milvus import MilvusStore
+from application.vectorstore.mongodb import MongoDBVectorStore
+from application.vectorstore.qdrant import QdrantStore
 
 
 class VectorCreator:
     vectorstores = {
-        'faiss': FaissStore,
-        'elasticsearch':ElasticsearchStore
+        "faiss": FaissStore,
+        "elasticsearch": ElasticsearchStore,
+        "mongodb": MongoDBVectorStore,
+        "qdrant": QdrantStore,
+        "milvus": MilvusStore,
     }
 
     @classmethod
@@ -13,4 +19,4 @@ class VectorCreator:
         vectorstore_class = cls.vectorstores.get(type.lower())
         if not vectorstore_class:
             raise ValueError(f"No vectorstore class found for type {type}")
-        return vectorstore_class(*args, **kwargs)
+        return vectorstore_class(*args, **kwargs)

application/worker.py

@@ -1,39 +1,111 @@
+import logging
 import os
 import shutil
 import string
 import zipfile
+from collections import Counter
 from urllib.parse import urljoin
 
-import nltk
 import requests
+from bson.objectid import ObjectId
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 from application.parser.file.bulk import SimpleDirectoryReader
 from application.parser.open_ai_func import call_openai_api
+from application.parser.remote.remote_creator import RemoteCreator
 from application.parser.schema.base import Document
 from application.parser.token_func import group_split
+from application.utils import count_tokens_docs
 
-try:
-    nltk.download('punkt', quiet=True)
-    nltk.download('averaged_perceptron_tagger', quiet=True)
-except FileExistsError:
-    pass
+mongo = MongoDB.get_client()
+db = mongo["docsgpt"]
+sources_collection = db["sources"]
 
+# Constants
+MIN_TOKENS = 150
+MAX_TOKENS = 1250
+RECURSION_DEPTH = 2
 
 # Define a function to extract metadata from a given filename.
 def metadata_from_filename(title):
-    store = '/'.join(title.split('/')[1:3])
-    return {'title': title, 'store': store}
-
+    return {"title": title}
 
 # Define a function to generate a random string of a given length.
 def generate_random_string(length):
-    return ''.join([string.ascii_letters[i % 52] for i in range(length)])
+    return "".join([string.ascii_letters[i % 52] for i in range(length)])
 
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
+    """
+    Recursively extract zip files with a limit on recursion depth.
+
+    Args:
+        zip_path (str): Path to the zip file to be extracted.
+        extract_to (str): Destination path for extracted files.
+        current_depth (int): Current depth of recursion.
+        max_depth (int): Maximum allowed depth of recursion to prevent infinite loops.
+    """
+    if current_depth > max_depth:
+        logging.warning(f"Reached maximum recursion depth of {max_depth}")
+        return
+
+    try:
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(extract_to)
+        os.remove(zip_path)  # Remove the zip file after extracting
+    except Exception as e:
+        logging.error(f"Error extracting zip file {zip_path}: {e}")
+        return
+
+    # Check for nested zip files and extract them
+    for root, dirs, files in os.walk(extract_to):
+        for file in files:
+            if file.endswith(".zip"):
+                # If a nested zip file is found, extract it recursively
+                file_path = os.path.join(root, file)
+                extract_zip_recursive(file_path, root, current_depth + 1, max_depth)
+
+def download_file(url, params, dest_path):
+    try:
+        response = requests.get(url, params=params)
+        response.raise_for_status()
+        with open(dest_path, "wb") as f:
+            f.write(response.content)
+    except requests.RequestException as e:
+        logging.error(f"Error downloading file: {e}")
+        raise
+
+def upload_index(full_path, file_data):
+    try:
+        if settings.VECTOR_STORE == "faiss":
+            files = {
+                "file_faiss": open(full_path + "/index.faiss", "rb"),
+                "file_pkl": open(full_path + "/index.pkl", "rb"),
+            }
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
+            )
+        else:
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), data=file_data
+            )
+        response.raise_for_status()
+    except requests.RequestException as e:
+        logging.error(f"Error uploading index: {e}")
+        raise
+    finally:
+        if settings.VECTOR_STORE == "faiss":
+            for file in files.values():
+                file.close()
 
 # Define the main function for ingesting and processing documents.
-def ingest_worker(self, directory, formats, name_job, filename, user):
+def ingest_worker(
+    self, directory, formats, name_job, filename, user, retriever="classic"
+):
     """
     Ingest and process documents.
 
@@ -44,80 +116,189 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         name_job (str): Name of the job for this ingestion task.
         filename (str): Name of the file to be ingested.
         user (str): Identifier for the user initiating the ingestion.
+        retriever (str): Type of retriever to use for processing the documents.
 
     Returns:
         dict: Information about the completed ingestion task, including input parameters and a "limited" flag.
     """
-    # directory = 'inputs' or 'temp'
-    # formats = [".rst", ".md"]
     input_files = None
     recursive = True
     limit = None
     exclude = True
-    # name_job = 'job1'
-    # filename = 'install.rst'
-    # user = 'local'
     sample = False
     token_check = True
-    min_tokens = 150
-    max_tokens = 1250
-    full_path = directory + '/' + user + '/' + name_job
-    import sys
-    print(full_path, file=sys.stderr)
-    # check if API_URL env variable is set
-    file_data = {'name': name_job, 'file': filename, 'user': user}
-    response = requests.get(urljoin(settings.API_URL, "/api/download"), params=file_data)
-    # check if file is in the response
-    print(response, file=sys.stderr)
-    file = response.content
+    full_path = os.path.join(directory, user, name_job)
+
+    logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job})
+    file_data = {"name": name_job, "file": filename, "user": user}
 
     if not os.path.exists(full_path):
         os.makedirs(full_path)
-    with open(full_path + '/' + filename, 'wb') as f:
-        f.write(file)
+    download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename))
 
     # check if file is .zip and extract it
-    if filename.endswith('.zip'):
-        with zipfile.ZipFile(full_path + '/' + filename, 'r') as zip_ref:
-            zip_ref.extractall(full_path)
-        os.remove(full_path + '/' + filename)
+    if filename.endswith(".zip"):
+        extract_zip_recursive(
+            os.path.join(full_path, filename), full_path, 0, RECURSION_DEPTH
+        )
 
-    self.update_state(state='PROGRESS', meta={'current': 1})
+    self.update_state(state="PROGRESS", meta={"current": 1})
 
-    raw_docs = SimpleDirectoryReader(input_dir=full_path, input_files=input_files, recursive=recursive,
-                                     required_exts=formats, num_files_limit=limit,
-                                     exclude_hidden=exclude, file_metadata=metadata_from_filename).load_data()
-    raw_docs = group_split(documents=raw_docs, min_tokens=min_tokens, max_tokens=max_tokens, token_check=token_check)
+    raw_docs = SimpleDirectoryReader(
+        input_dir=full_path,
+        input_files=input_files,
+        recursive=recursive,
+        required_exts=formats,
+        num_files_limit=limit,
+        exclude_hidden=exclude,
+        file_metadata=metadata_from_filename,
+    ).load_data()
+    raw_docs = group_split(
+        documents=raw_docs,
+        min_tokens=MIN_TOKENS,
+        max_tokens=MAX_TOKENS,
+        token_check=token_check,
+    )
 
     docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+    id = ObjectId()
 
-    call_openai_api(docs, full_path, self)
-    self.update_state(state='PROGRESS', meta={'current': 100})
+    call_openai_api(docs, full_path, id, self)
+    tokens = count_tokens_docs(docs)
+    self.update_state(state="PROGRESS", meta={"current": 100})
 
     if sample:
         for i in range(min(5, len(raw_docs))):
-            print(raw_docs[i].text)
+            logging.info(f"Sample document {i}: {raw_docs[i]}")
 
-    # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
-    # and send them to the server (provide user and name in form)
-    file_data = {'name': name_job, 'user': user}
-    if settings.VECTOR_STORE == "faiss":
-        files = {'file_faiss': open(full_path + '/index.faiss', 'rb'),
-                'file_pkl': open(full_path + '/index.pkl', 'rb')}
-        response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data)
-        response = requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path))
-    else:
-        response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
+    file_data.update({
+        "tokens": tokens,
+        "retriever": retriever,
+        "id": str(id),
+        "type": "local",
+    })
+    upload_index(full_path, file_data)
 
-    
     # delete local
     shutil.rmtree(full_path)
 
     return {
-        'directory': directory,
-        'formats': formats,
-        'name_job': name_job,
-        'filename': filename,
-        'user': user,
-        'limited': False
+        "directory": directory,
+        "formats": formats,
+        "name_job": name_job,
+        "filename": filename,
+        "user": user,
+        "limited": False,
+    }
+
+def remote_worker(
+    self,
+    source_data,
+    name_job,
+    user,
+    loader,
+    directory="temp",
+    retriever="classic",
+    sync_frequency="never",
+    operation_mode="upload",
+    doc_id=None,
+):
+    token_check = True
+    full_path = os.path.join(directory, user, name_job)
+
+    if not os.path.exists(full_path):
+        os.makedirs(full_path)
+    self.update_state(state="PROGRESS", meta={"current": 1})
+    logging.info(
+        f"Remote job: {full_path}",
+        extra={"user": user, "job": name_job, "source_data": source_data},
+    )
+
+    remote_loader = RemoteCreator.create_loader(loader)
+    raw_docs = remote_loader.load_data(source_data)
+
+    docs = group_split(
+        documents=raw_docs,
+        min_tokens=MIN_TOKENS,
+        max_tokens=MAX_TOKENS,
+        token_check=token_check,
+    )
+    tokens = count_tokens_docs(docs)
+    if operation_mode == "upload":
+        id = ObjectId()
+        call_openai_api(docs, full_path, id, self)
+    elif operation_mode == "sync":
+        if not doc_id or not ObjectId.is_valid(doc_id):
+            raise ValueError("doc_id must be provided for sync operation.")
+        id = ObjectId(doc_id)
+        call_openai_api(docs, full_path, id, self)
+    self.update_state(state="PROGRESS", meta={"current": 100})
+
+    file_data = {
+        "name": name_job,
+        "user": user,
+        "tokens": tokens,
+        "retriever": retriever,
+        "id": str(id),
+        "type": loader,
+        "remote_data": source_data,
+        "sync_frequency": sync_frequency,
+    }
+    upload_index(full_path, file_data)
+
+    shutil.rmtree(full_path)
+
+    return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}
+
+def sync(
+    self,
+    source_data,
+    name_job,
+    user,
+    loader,
+    sync_frequency,
+    retriever,
+    doc_id=None,
+    directory="temp",
+):
+    try:
+        remote_worker(
+            self,
+            source_data,
+            name_job,
+            user,
+            loader,
+            directory,
+            retriever,
+            sync_frequency,
+            "sync",
+            doc_id,
+        )
+    except Exception as e:
+        logging.error(f"Error during sync: {e}")
+        return {"status": "error", "error": str(e)}
+    return {"status": "success"}
+
+def sync_worker(self, frequency):
+    sync_counts = Counter()
+    sources = sources_collection.find()
+    for doc in sources:
+        if doc.get("sync_frequency") == frequency:
+            name = doc.get("name")
+            user = doc.get("user")
+            source_type = doc.get("type")
+            source_data = doc.get("remote_data")
+            retriever = doc.get("retriever")
+            doc_id = str(doc.get("_id"))
+            resp = sync(
+                self, source_data, name, user, source_type, frequency, retriever, doc_id
+            )
+            sync_counts["total_sync_count"] += 1
+            sync_counts[
+                "sync_success" if resp["status"] == "success" else "sync_failure"
+            ] += 1
+
+    return {
+        key: sync_counts[key]
+        for key in ["total_sync_count", "sync_success", "sync_failure"]
     }

application/wsgi.py

@@ -1,4 +1,5 @@
 from application.app import app
+from application.core.settings import settings
 
 if __name__ == "__main__":
-    app.run(debug=True, port=7091)
+    app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)

docker-compose-azure.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docker-compose-dev.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
 
   redis:

docker-compose-local.yaml

@@ -1,8 +1,8 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend
+    volumes:
+    - ./frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING

docker-compose-mock.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docker-compose.yaml

@@ -1,8 +1,8 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend
+    volumes:
+      - ./frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING
@@ -20,6 +20,7 @@ services:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - CACHE_REDIS_URL=redis://redis:6379/2
     ports:
       - "7091:7091"
     volumes:
@@ -32,7 +33,7 @@ services:
 
   worker:
     build: ./application
-    command: celery -A application.app.celery worker -l INFO
+    command: celery -A application.app.celery worker -l INFO -B
     environment:
       - API_KEY=$API_KEY
       - EMBEDDINGS_KEY=$API_KEY
@@ -41,6 +42,7 @@ services:
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
+      - CACHE_REDIS_URL=redis://redis:6379/2
     depends_on:
       - redis
       - mongo

docs/README.md

@@ -46,6 +46,6 @@ yarn install
 yarn dev
 ```
 
-- Now, you should be able to view the docs on your local environment by visiting `http://localhost:5000`. You can explore the different markdown files and make changes as you see fit.
+- Now, you should be able to view the docs on your local environment by visiting `http://localhost:3000`. You can explore the different markdown files and make changes as you see fit.
 
 - **Footnotes:** This guide assumes you have Node.js and npm installed. The guide involves running a local server using yarn, and viewing the documentation offline. If you encounter any issues, it may be worth verifying your Node.js and npm installations and whether you have installed yarn correctly.

docs/next.config.js

@@ -1,9 +1,9 @@
 const withNextra = require('nextra')({
-    theme: 'nextra-theme-docs',
-    themeConfig: './theme.config.jsx'
-  })
+  theme: 'nextra-theme-docs',
+  themeConfig: './theme.config.jsx'
+})
+
+module.exports = withNextra()
    
-  module.exports = withNextra()
-   
-  // If you have other Next.js configurations, you can pass them as the parameter:
-  // module.exports = withNextra({ /* other next.js config */ })
+// If you have other Next.js configurations, you can pass them as the parameter:
+// module.exports = withNextra({ /* other next.js config */ })

docs/package.json

@@ -1,17 +1,17 @@
 {
-"scripts":{
-      "dev": "next dev",
+  "scripts": {
+    "dev": "next dev",
     "build": "next build",
     "start": "next start"
   },
-    "license": "MIT",
+  "license": "MIT",
   "dependencies": {
-    "@vercel/analytics": "^1.0.2",
-    "docsgpt": "^0.2.4",
-    "next": "^13.4.19",
-    "nextra": "^2.12.3",
-    "nextra-theme-docs": "^2.12.3",
+    "@vercel/analytics": "^1.1.1",
+    "docsgpt": "^0.4.7",
+    "next": "^14.2.12",
+    "nextra": "^2.13.2",
+    "nextra-theme-docs": "^2.13.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0"
   }
-}
+}

docs/pages/API/API-docs.md

@@ -227,3 +227,124 @@ JSON response indicating the status of the operation:
 ```json
 { "status": "ok" }
 ```
+
+### 7. /api/get_api_keys
+**Description:**
+
+The endpoint retrieves a list of API keys for the user.
+
+**Request:**
+
+**Method**: `GET`
+
+**Sample JavaScript Fetch Request:**
+```js
+// get_api_keys (GET http://127.0.0.1:5000/api/get_api_keys)
+fetch("http://localhost:5001/api/get_api_keys", {
+      "method": "GET",
+      "headers": {
+            "Content-Type": "application/json; charset=utf-8"
+      },
+})
+.then((res) => res.text())
+.then(console.log.bind(console))
+
+```
+**Response:**
+
+JSON response with a list of created API keys:
+
+```json
+[
+      {
+        "id": "string",
+        "name": "string",
+        "key": "string",
+        "source": "string"
+      },
+      ...
+    ]
+```
+
+### 8. /api/create_api_key
+
+**Description:**
+
+Create a new API key for the user.
+
+**Request:**
+
+**Method**: `POST`
+
+**Headers**: Content-Type should be set to `application/json; charset=utf-8`
+
+**Request Body**: JSON object with the following fields:
+* `name` — A name for the API key.
+* `source` — The source documents that will be used.
+* `prompt_id` — The prompt ID.
+* `chunks` — The number of chunks used to process an answer.
+
+Here is a JavaScript Fetch Request example:
+```js
+// create_api_key (POST http://127.0.0.1:5000/api/create_api_key)
+fetch("http://127.0.0.1:5000/api/create_api_key", {
+      "method": "POST",
+      "headers": {
+            "Content-Type": "application/json; charset=utf-8"
+      },
+      "body": JSON.stringify({"name":"Example Key Name",
+          "source":"Example Source",
+          "prompt_id":"creative",
+          "chunks":"2"})
+})
+.then((res) => res.json())
+.then(console.log.bind(console))
+```
+
+**Response**
+
+In response, you will get a JSON document containing the `id` and `key`:
+```json
+{
+  "id": "string",
+  "key": "string"
+}
+```
+
+### 9. /api/delete_api_key
+
+**Description:**
+
+Delete an API key for the user.
+
+**Request:**
+
+**Method**: `POST`
+
+**Headers**: Content-Type should be set to `application/json; charset=utf-8`
+
+**Request Body**: JSON object with the field:
+* `id` — The unique identifier of the API key to be deleted.
+
+Here is a JavaScript Fetch Request example:
+```js
+// delete_api_key (POST http://127.0.0.1:5000/api/delete_api_key)
+fetch("http://127.0.0.1:5000/api/delete_api_key", {
+      "method": "POST",
+      "headers": {
+            "Content-Type": "application/json; charset=utf-8"
+      },
+      "body": JSON.stringify({"id":"API_KEY_ID"})
+})
+.then((res) => res.json())
+.then(console.log.bind(console))
+```
+
+**Response:**
+
+In response, you will get a JSON document indicating the status of the operation:
+```json
+{
+  "status": "ok"
+}
+```

docs/pages/API/_meta.json

@@ -0,0 +1,10 @@
+{
+  "API-docs": {
+    "title": "🗂️️ API-docs",
+    "href": "/API/API-docs"
+  },
+  "api-key-guide": {
+    "title": "🔐 API Keys guide",
+    "href": "/API/api-key-guide"
+  }
+}

docs/pages/API/api-key-guide.md

@@ -0,0 +1,30 @@
+## Guide to DocsGPT API Keys
+
+DocsGPT API keys are essential for developers and users who wish to integrate the  DocsGPT models into external applications, such as the our widget. This guide will walk you through the steps of obtaining an API key, starting from uploading your document to understanding the key variables associated with API keys.
+
+### Uploading Your Document
+
+Before creating your first API key, you must upload the document that will be linked to this key. You can upload your document through two methods:
+
+- **GUI Web App Upload:** A user-friendly graphical interface that allows for easy upload and management of documents.
+- **Using `/api/upload` Method:** For users comfortable with API calls, this method provides a direct way to upload documents.
+
+### Obtaining Your API Key
+
+After uploading your document, you can obtain an API key either through the graphical user interface or via an API call:
+
+- **Graphical User Interface:** Navigate to the Settings section of the DocsGPT web app, find the API Keys option, and press 'Create New' to generate your key.
+- **API Call:** Alternatively, you can use the `/api/create_api_key` endpoint to create a new API key. For detailed instructions, visit [DocsGPT API Documentation](https://docs.docsgpt.cloud/API/API-docs#8-apicreate_api_key).
+
+### Understanding Key Variables
+
+Upon creating your API key, you will encounter several key variables. Each serves a specific purpose:
+
+- **Name:** Assign a name to your API key for easy identification.
+- **Source:** Indicates the source document(s) linked to your API key, which DocsGPT will use to generate responses.
+- **ID:** A unique identifier for your API key. You can view this by making a call to `/api/get_api_keys`.
+- **Key:** The API key itself, which will be used in your application to authenticate API requests.
+
+With your API key ready, you can now integrate DocsGPT into your application, such as the DocsGPT Widget or any other software, via `/api/answer` or `/stream` endpoints. The source document is preset with the API key, allowing you to bypass fields like `selectDocs` and `active_docs` during implementation.
+
+Congratulations on taking the first step towards enhancing your applications with DocsGPT! With this guide, you're now equipped to navigate the process of obtaining and understanding DocsGPT API keys.

docs/pages/Deploying/Hosting-the-app.md

@@ -107,3 +107,4 @@ Your instance is now available at your Public IP Address on port 5173. Enjoy usi
 
 - [Deploy DocsGPT on Civo Compute Cloud](https://dev.to/rutamhere/deploying-docsgpt-on-civo-compute-c)
 - [Deploy DocsGPT on DigitalOcean Droplet](https://dev.to/rutamhere/deploying-docsgpt-on-digitalocean-droplet-50ea)
+- [Deploy DocsGPT on Kamatera Performance Cloud](https://dev.to/rutamhere/deploying-docsgpt-on-kamatera-performance-cloud-1bj)

docs/pages/Deploying/Kubernetes-Deploying.md

@@ -0,0 +1,100 @@
+# Self-hosting DocsGPT on Kubernetes
+
+This guide will walk you through deploying DocsGPT on Kubernetes.
+
+## Prerequisites
+
+Ensure you have the following installed before proceeding:
+
+- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
+- Access to a Kubernetes cluster
+
+## Folder Structure
+
+The `k8s` folder contains the necessary deployment and service configuration files:
+
+- `deployments/`
+- `services/`
+- `docsgpt-secrets.yaml`
+
+## Deployment Instructions
+
+1. **Clone the Repository**
+
+   ```sh
+   git clone https://github.com/arc53/DocsGPT.git
+   cd docsgpt/k8s
+   ```
+
+2. **Configure Secrets (optional)**
+
+   Ensure that you have all the necessary secrets in `docsgpt-secrets.yaml`. Update it with your secrets before applying if you want. By default we will use qdrant as a vectorstore and public docsgpt llm as llm for inference.
+
+3. **Apply Kubernetes Deployments**
+
+   Deploy your DocsGPT resources using the following commands:
+
+   ```sh
+   kubectl apply -f deployments/
+   ```
+
+4. **Apply Kubernetes Services**
+
+   Set up your services using the following commands:
+
+   ```sh
+   kubectl apply -f services/
+   ```
+
+5. **Apply Secrets**
+
+   Apply the secret configurations:
+
+   ```sh
+   kubectl apply -f docsgpt-secrets.yaml
+   ```
+
+6. **Substitute API URL**
+
+   After deploying the services, you need to update the environment variable `VITE_API_HOST` in your deployment file `deployments/docsgpt-deploy.yaml` with the actual endpoint URL created by your `docsgpt-api-service`.
+
+    ```sh
+    kubectl get services/docsgpt-api-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}' | xargs -I {} sed -i "s|<your-api-endpoint>|{}|g" deployments/docsgpt-deploy.yaml
+    ```
+
+7. **Rerun Deployment**
+
+   After making the changes, reapply the deployment configuration to update the environment variables:
+
+   ```sh
+   kubectl apply -f deployments/
+   ```
+
+## Verifying the Deployment
+
+To verify if everything is set up correctly, you can run the following:
+
+```sh
+kubectl get pods
+kubectl get services
+```
+
+Ensure that the pods are running and the services are available.
+
+## Accessing DocsGPT
+
+To access DocsGPT, you need to find the external IP address of the frontend service. You can do this by running:
+
+```sh
+kubectl get services/docsgpt-frontend-service | awk 'NR>1 {print "http://" $4}'
+```
+
+## Troubleshooting
+
+If you encounter any issues, you can check the logs of the pods for more details:
+
+```sh
+kubectl logs <pod-name>
+```
+
+Replace `<pod-name>` with the actual name of your DocsGPT pod.

docs/pages/Deploying/Quickstart.md

@@ -8,7 +8,7 @@ Just run the following command:
 ./setup.sh
 ```
 
-This command will install all the necessary dependencies and provide you with an option to download the local model or use OpenAI.
+This command will install all the necessary dependencies and provide you with an option to use our LLM API, download the local model or use OpenAI.
 
 If you prefer to follow manual steps, refer to this guide:
 
@@ -16,7 +16,7 @@ If you prefer to follow manual steps, refer to this guide:
    ```bash
    git clone https://github.com/arc53/DocsGPT.git
    ```
-2. Create a `.env` file in your root directory and set your `API_KEY` with your [OpenAI API key](https://platform.openai.com/account/api-keys).
+2. Create a `.env` file in your root directory and set your `API_KEY` with your [OpenAI API key](https://platform.openai.com/account/api-keys). (optional in case you want to use OpenAI)
 3. Run the following commands:
    ```bash
    docker-compose build && docker-compose up
@@ -67,62 +67,3 @@ To run the setup on Windows, you have two options: using the Windows Subsystem f
 
 These steps should help you set up and run the project on Windows using either WSL or Git Bash/Command Prompt. 
 **Important:** Ensure that Docker is installed and properly configured on your Windows system for these steps to work.
-
-
-For WINDOWS:
-
-To run the given setup on Windows, you can use the Windows Subsystem for Linux (WSL) or a Git Bash terminal to execute similar commands. Here are the steps adapted for Windows:
-
-Option 1: Using Windows Subsystem for Linux (WSL):
-
-1. Install WSL if you haven't already. You can follow the official Microsoft documentation for installation: (https://learn.microsoft.com/en-us/windows/wsl/install).
-2. After setting up WSL, open the WSL terminal.
-3. Clone the repository and create the `.env` file:
-   ```bash
-   git clone https://github.com/arc53/DocsGPT.git
-   cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
-   ```
-4. Run the following command to start the setup with Docker Compose:
-    ```bash
-    ./run-with-docker-compose.sh
-    ```
-5. Open your web browser and navigate to http://localhost:5173/.
-6. To stop the setup, just press **Ctrl + C** in the WSL terminal.
-
-Option 2: Using Git Bash or Command Prompt (CMD):
-
-1. Install Git for Windows if you haven't already. You can download it from the official website: (https://gitforwindows.org/).
-2. Open Git Bash or Command Prompt.
-3. Clone the repository and create the `.env` file:
-   ```bash
-   git clone https://github.com/arc53/DocsGPT.git
-   cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
-   ```
-4. Run the following command to start the setup with Docker Compose:
-   ```bash
-   ./run-with-docker-compose.sh
-   ```
-5. Open your web browser and navigate to http://localhost:5173/.
-6. To stop the setup, just press **Ctrl + C** in the Git Bash or Command Prompt terminal.
-
-These steps should help you set up and run the project on Windows using either WSL or Git Bash/Command Prompt. Make sure you have Docker installed and properly configured on your Windows system for this to work.
-
-
-### Chrome Extension
-
-#### Installing the Chrome extension:
-To enhance your DocsGPT experience, you can install the DocsGPT Chrome extension. Here's how:
-
-1. In the DocsGPT GitHub repository, click on the **Code** button and select **Download ZIP**.
-2. Unzip the downloaded file to a location you can easily access.
-3. Open the Google Chrome browser and click on the three dots menu (upper right corner).
-4. Select **More Tools** and then **Extensions**.
-5. Turn on the **Developer mode** switch in the top right corner of the **Extensions page**.
-6. Click on the **Load unpacked** button.
-7. Select the **Chrome** folder where the DocsGPT files have been unzipped (docsgpt-main > extensions > chrome).
-8. The extension should now be added to Google Chrome and can be managed on the Extensions page.
-9. To disable or remove the extension, simply turn off the toggle switch on the extension card or click the **Remove** button.

docs/pages/Deploying/_meta.json

@@ -10,5 +10,9 @@
   "Railway-Deploying": {
     "title": "🚂Deploying on Railway",
     "href": "/Deploying/Railway-Deploying"
+  },
+  "Kubernetes-Deploying": {
+    "title": "☸️Deploying on Kubernetes",
+    "href": "/Deploying/Kubernetes-Deploying"
   }
-}
+}

docs/pages/Developing/_meta.json

@@ -1,6 +0,0 @@
-{
-  "API-docs": {
-    "title": "🗂️️ API-docs",
-    "href": "/Developing/API-docs"
-  }
-}

docs/pages/Extensions/Chrome-extension.mdx

@@ -0,0 +1,34 @@
+
+import {Steps} from 'nextra/components'
+import { Callout } from 'nextra/components'
+
+
+## Chrome  Extension Setup Guide
+
+To enhance your DocsGPT experience, you can install the DocsGPT Chrome extension. Here's how:
+<Steps >
+### Step 1
+
+
+
+In the DocsGPT GitHub repository, click on the **Code** button and select **Download ZIP**.
+### Step 2
+Unzip the downloaded file to a location you can easily access.
+### Step 3
+Open the Google Chrome browser and click on the three dots menu (upper right corner).
+### Step 4
+Select **More Tools** and then **Extensions**.
+### Step 5
+Turn on the **Developer mode** switch in the top right corner of the **Extensions page**.
+### Step 6
+Click on the **Load unpacked** button.
+### Step 7
+7. Select the **Chrome** folder where the DocsGPT files have been unzipped (docsgpt-main > extensions > chrome).
+### Step 8
+The extension should now be added to Google Chrome and can be managed on the Extensions page.
+### Step 9
+ To disable or remove the extension, simply turn off the toggle switch on the extension card or click the **Remove** button.
+</Steps>
+
+
+

docs/pages/Extensions/_meta.json

@@ -4,7 +4,11 @@
     "href": "/Extensions/Chatwoot-extension"
   },
   "react-widget": {
-      "title": "🏗️ Widget setup",
-      "href": "/Extensions/react-widget"
-    }
+    "title": "🏗️ Widget setup",
+    "href": "/Extensions/react-widget"
+  },
+  "Chrome-extension": {
+    "title": "🌐 Chrome Extension",
+    "href": "/Extensions/Chrome-extension"
+  }
 }