Merge pull request #1608 from siiddhantt/feat/api-tool

feat: API Tool

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: arc53

.github/dependabot.yml

@@ -0,0 +1,19 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/application" # Location of package manifests
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm" # See documentation for possible values
+    directory: "/frontend" # Location of package manifests
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/holopin.yml

@@ -1,5 +1,11 @@
-organization: arc53
-defaultSticker: clqmdf0ed34290glbvqh0kzxd
+organization: docsgpt
+defaultSticker: cm1ulwkkl180570cl82rtzympu
 stickers:
-  - id: clqmdf0ed34290glbvqh0kzxd
-    alias: festive
+  - id: cm1ulwkkl180570cl82rtzympu
+    alias: contributor2024
+  - id: cm1ureg8o130450cl8c1po6mil
+    alias: api
+  - id: cm1urhmag148240cl8yvqxkthx
+    alias: lpc
+  - id: cm1urlcpq622090cl2tvu4w71y
+    alias: lexeu

.github/labeler.yml

@@ -1,23 +1,31 @@
 repo:
-  - '*'
+- changed-files:
+  - any-glob-to-any-file: '*'
 
 github:
-  - .github/**/*
+- changed-files:
+  - any-glob-to-any-file: '.github/**/*'
 
 application:
-  - application/**/*
+- changed-files:
+  - any-glob-to-any-file: 'application/**/*'
 
 docs:
-  - docs/**/*
+- changed-files:
+  - any-glob-to-any-file: 'docs/**/*'
 
 extensions:
-  - extensions/**/*
+- changed-files:
+  - any-glob-to-any-file: 'extensions/**/*'
 
 frontend:
-  - frontend/**/*
+- changed-files:
+  - any-glob-to-any-file: 'frontend/**/*'
 
 scripts:
-  - scripts/**/*
+- changed-files:
+  - any-glob-to-any-file: 'scripts/**/*'
 
 tests:
-  - tests/**/*
+- changed-files:
+  - any-glob-to-any-file: 'tests/**/*'

.github/workflows/ci.yml

@@ -1,10 +1,8 @@
 name: Build and push DocsGPT Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
@@ -14,34 +12,36 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           file: './application/Dockerfile'
           platforms: linux/amd64
           context: ./application
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+          cache-to: type=inline

.github/workflows/cife.yml

@@ -1,10 +1,8 @@
 name: Build and push DocsGPT-FE Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
@@ -14,22 +12,22 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -37,12 +35,14 @@ jobs:
 
       # Runs a single command using the runners shell
       - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           file: './frontend/Dockerfile'
           platforms: linux/amd64, linux/arm64
           context: ./frontend
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+          cache-to: type=inline

.github/workflows/docker-develop-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v6
+        with:
+          file: './application/Dockerfile'
+          platforms: linux/amd64
+          context: ./application
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+          cache-to: type=inline

.github/workflows/docker-develop-fe-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT FE Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v6
+        with:
+          file: './frontend/Dockerfile'
+          platforms: linux/amd64
+          context: ./frontend
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+          cache-to: type=inline

.github/workflows/labeler.yml

@@ -4,12 +4,13 @@ on:
   - pull_request_target
 jobs:
   triage:
+    if: github.repository == 'arc53/DocsGPT'
     permissions:
       contents: read
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/labeler@v4
+      - uses: actions/labeler@v5
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           sync-labels: true

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Lint with Ruff
         uses: chartboost/ruff-action@v1

.github/workflows/pytest.yml

@@ -6,11 +6,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.12"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -21,10 +21,10 @@ jobs:
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
       - name: Test with pytest and generate coverage report
         run: |
-          python -m pytest --cov=application --cov=scripts --cov=extensions --cov-report=xml
+          python -m pytest --cov=application --cov-report=xml
       - name: Upload coverage reports to Codecov
-        if: github.event_name == 'pull_request' && matrix.python-version == '3.11'
-        uses: codecov/codecov-action@v3
+        if: github.event_name == 'pull_request' && matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/sync_fork.yaml

@@ -17,7 +17,7 @@ jobs:
     steps:
       # Step 1: run a standard checkout action
       - name: Checkout target repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Step 2: run the sync action
       - name: Sync upstream changes

.gitignore

@@ -172,5 +172,5 @@ application/vectors/
 
 node_modules/
 .vscode/settings.json
-models/
+/models/
 model/

.vscode/launch.json

@@ -0,0 +1,54 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Docker Debug Frontend",
+      "request": "launch",
+      "type": "chrome",
+      "preLaunchTask": "docker-compose: debug:frontend",
+      "url": "http://127.0.0.1:5173",
+      "webRoot": "${workspaceFolder}/frontend",
+      "skipFiles": [
+        "<node_internals>/**"
+      ]
+    },
+    {
+        "name": "Flask Debugger",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "flask",
+        "env": {
+            "FLASK_APP": "application/app.py",
+            "PYTHONPATH": "${workspaceFolder}",
+            "FLASK_ENV": "development",
+            "FLASK_DEBUG": "1",
+            "FLASK_RUN_PORT": "7091",
+            "FLASK_RUN_HOST": "0.0.0.0"
+
+        },
+        "args": [
+            "run",
+            "--no-debugger"
+        ],
+        "cwd": "${workspaceFolder}",
+    },
+    {
+      "name": "Celery Debugger",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}",
+      },
+      "args": [
+        "-A",
+        "application.app.celery",
+        "worker",
+        "-l",
+        "INFO",
+        "--pool=solo"
+      ],
+      "cwd": "${workspaceFolder}"
+    }
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,21 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "docker-compose",
+      "label": "docker-compose: debug:frontend",
+      "dockerCompose": {
+        "up": {
+          "detached": true,
+          "services": [
+            "frontend"
+          ],
+          "build": true
+        },
+        "files": [
+          "${workspaceFolder}/docker-compose.yaml"
+        ]
+      }
+    }
+  ]
+}

CONTRIBUTING.md

@@ -6,7 +6,7 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 
 📣 **Discussions** - Engage in conversations, start new topics, or help answer questions.
 
-🐞 **Issues** - This is where we keep track of tasks. It could be bugs,fixes or suggestions for new features.
+🐞 **Issues** - This is where we keep track of tasks. It could be bugs, fixes or suggestions for new features.
 
 🛠️ **Pull requests** - Suggest changes to our repository, either by working on existing issues or adding new features.
 
@@ -21,11 +21,13 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 - If you're interested in contributing code, here are some important things to know:
 
 - We have a frontend built on React (Vite) and a backend in Python.
-=======
-Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version that you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
+
+  
+Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
 
 ### 👨‍💻 If you're interested in contributing code, here are some important things to know:
 
+For instructions on setting up a development environment, please refer to our [Development Deployment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).
 
 Tech Stack Overview:
 
@@ -35,15 +37,14 @@ Tech Stack Overview:
 
 ### 🌐 If you are looking to contribute to frontend (⚛️React, Vite):
 
-- The current frontend is being migrated from [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) to [`/frontend`](https://github.com/arc53/DocsGPT/tree/main/frontend) with a new design, so please contribute to the new one.
-- Check out this [milestone](https://github.com/arc53/DocsGPT/milestone/1) and its issues.
+
 - The updated Figma design can be found [here](https://www.figma.com/file/OXLtrl1EAy885to6S69554/DocsGPT?node-id=0%3A1&t=hjWVuxRg9yi5YkJ9-1).
 
 Please try to follow the guidelines.
 
 ### 🖥 If you are looking to contribute to Backend (🐍 Python):
 
-- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) or [`/scripts`](https://github.com/arc53/DocsGPT/tree/main/scripts) (please disregard old [`ingest_rst.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst.py) [`ingest_rst_sphinx.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst_sphinx.py) files; they will be deprecated soon).
+- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) 
 - All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
 - Before submitting your Pull Request, ensure it can be queried after ingesting some test data.
   
@@ -125,4 +126,4 @@ Thank you for considering contributing to DocsGPT! 🙏
 
 ## Questions/collaboration
 Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU). We're very friendly and welcoming to new contributors, so don't hesitate to reach out.
-# Thank you so much for considering to contribute DocsGPT!🙏
+# Thank you so much for considering to contributing DocsGPT!🙏

README.md

@@ -3,13 +3,11 @@
 </h1>
 
 <p align="center">
-  <strong>Open-Source Documentation Assistant</strong>
+  <strong>Open-Source RAG Assistant</strong>
 </p>
 
 <p align="left">
-  <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> is a cutting-edge open-source solution that streamlines the process of finding information in the project documentation. With its integration of the powerful <strong>GPT</strong> models, developers can easily ask questions about a project and receive accurate answers.
-  
-Say goodbye to time-consuming manual searches, and let <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> help you quickly find the information you need. Try it out and see how it revolutionizes your project documentation experience. Contribute to its development and be a part of the future of AI-powered assistance.
+  <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> is an open-source genAI tool that helps users get reliable answers from any knowledge source, while avoiding hallucinations. It enables quick and reliable information retrieval, with tooling and agentic system capability built in.
 </p>
 
 <div align="center">
@@ -20,173 +18,122 @@ Say goodbye to time-consuming manual searches, and let <strong><a href="https://
   <a href="https://discord.gg/n5BX8dh8rU">![link to discord](https://img.shields.io/discord/1070046503302877216)</a>
   <a href="https://twitter.com/docsgptai">![X (formerly Twitter) URL](https://img.shields.io/twitter/follow/docsgptai)</a>
 
- 
+  <br>
+
+  [☁️ Cloud Version](https://app.docsgpt.cloud/) • [💬 Discord](https://discord.gg/n5BX8dh8rU) • [📖 Guides](https://docs.docsgpt.cloud/)
+  <br>
+  [👫 Contribute](https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md) • [🏠 Self-host](https://docs.docsgpt.cloud/Guides/How-to-use-different-LLM) • [⚡️ Quickstart](https://github.com/arc53/DocsGPT#quickstart) 
+
 </div>
+<div align="center">
+<img src="https://d3dg1063dc54p9.cloudfront.net/videos/demov7.gif" alt="video-example-of-docs-gpt" width="800" height="450">
+</div>
+<h3 align="left">
+  <strong>Key Features:</strong>
+</h3>
+<ul align="left">
+    <li><strong>🗂️ Wide Format Support:</strong> Reads PDF, DOCX, CSV, XLSX, EPUB, MD, RST, HTML, MDX, JSON, PPTX, and images.</li>
+    <li><strong>🌐 Web & Data Integration:</strong> Ingests from URLs, sitemaps, Reddit, GitHub and web crawlers.</li>
+    <li><strong>✅ Reliable Answers:</strong> Get accurate, hallucination-free responses with source citations viewable in a clean UI.</li>
+    <li><strong>🔗 Actionable Tooling:</strong> Connect to APIs, tools, and other services to enable LLM actions.</li>
+    <li><strong>🧩 Pre-built Integrations:</strong> Use readily available HTML/React chat widgets, search tools, Discord/Telegram bots, and more.</li>
+    <li><strong>🔌 Flexible Deployment:</strong> Works with major LLMs (OpenAI, Google, Anthropic) and local models (Ollama, llama_cpp).</li>
+    <li><strong>🏢 Secure & Scalable:</strong> Run privately and securely with Kubernetes support, designed for enterprise-grade reliability.</li>
+</ul>
+
+## Roadmap
+
+- [x] Full GoogleAI compatibility (Jan 2025)
+- [x] Add tools (Jan 2025)
+- [ ] Anthropic Tool compatibility
+- [ ] Add triggerable actions / tools (webhook)
+- [ ] Add OAuth 2.0 authentication for tools and sources
+- [ ] Manually updating chunks in the app UI
+- [ ] Devcontainer for easy development
+- [ ] Chatbots menu re-design to handle tools, scheduling, and more
+
+You can find our full roadmap [here](https://github.com/orgs/arc53/projects/2). Please don't hesitate to contribute or create issues, it helps us improve DocsGPT!
 
 ### Production Support / Help for Companies:
 
 We're eager to provide personalized assistance when deploying your DocsGPT to a live environment.
 
-- [Get Enterprise / teams Demo :wave:](https://www.docsgpt.cloud/contact)
-- [Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
+[Get a Demo :wave:](https://www.docsgpt.cloud/contact)⁠
 
-![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
+[Send Email :email:](mailto:support@docsgpt.cloud?subject=DocsGPT%20support%2Fsolutions)
 
-## Roadmap
-
-You can find our roadmap [here](https://github.com/orgs/arc53/projects/2). Please don't hesitate to contribute or create issues, it helps us improve DocsGPT!
-
-## Our Open-Source Models Optimized for DocsGPT:
-
-| Name                                                                  | Base Model  | Requirements (or similar) |
-| --------------------------------------------------------------------- | ----------- | ------------------------- |
-| [Docsgpt-7b-mistral](https://huggingface.co/Arc53/docsgpt-7b-mistral)   | Mistral-7b   | 1xA10G gpu                |
-| [Docsgpt-14b](https://huggingface.co/Arc53/docsgpt-14b)               | llama-2-14b | 2xA10 gpu's               |
-| [Docsgpt-40b-falcon](https://huggingface.co/Arc53/docsgpt-40b-falcon) | falcon-40b  | 8xA10G gpu's              |
-
-If you don't have enough resources to run it, you can use bitsnbytes to quantize.
-
-## Features
-
-![Main features of DocsGPT showcasing six main features](https://user-images.githubusercontent.com/17906039/220427472-2644cff4-7666-46a5-819f-fc4a521f63c7.png)
-
-## Useful Links
-
-- :mag: :fire: [Cloud Version](https://app.docsgpt.cloud/)
-
-- :speech_balloon: :tada: [Join our Discord](https://discord.gg/n5BX8dh8rU)
-
-- :books: :sunglasses: [Guides](https://docs.docsgpt.cloud/)
-
-- :couple: [Interested in contributing?](https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md)
-
-- :file_folder: :rocket: [How to use any other documentation](https://docs.docsgpt.cloud/Guides/How-to-train-on-other-documentation)
-
-- :house: :closed_lock_with_key: [How to host it locally (so all data will stay on-premises)](https://docs.docsgpt.cloud/Guides/How-to-use-different-LLM)
-
-## Project Structure
-
-- Application - Flask app (main application).
-
-- Extensions - Chrome extension.
-
-- Scripts - Script that creates similarity search index for other libraries.
-
-- Frontend - Frontend uses <a href="https://vitejs.dev/">Vite</a> and <a href="https://react.dev/">React</a>.
 
 ## QuickStart
 
 > [!Note]
 > Make sure you have [Docker](https://docs.docker.com/engine/install/) installed
 
+
+1. Clone the repository and run the following command:
+  ```bash
+  git clone https://github.com/arc53/DocsGPT.git
+  cd DocsGPT
+  ```
+
 On Mac OS or Linux, write:
 
-`./setup.sh`
+
+2. Run the following command:
+  ```bash
+  ./setup.sh
+  ```
 
 It will install all the dependencies and allow you to download the local model, use OpenAI or use our LLM API.
 
 Otherwise, refer to this Guide for Windows:
 
-1. Download and open this repository with `git clone https://github.com/arc53/DocsGPT.git`
-2. Create a `.env` file in your root directory and set the env variables and `VITE_API_STREAMING` to true or false, depending on whether you want streaming answers or not.
+On windows:
+
+2. Create a `.env` file in your root directory and set the env variables.
    It should look like this inside:
 
    ```
    LLM_NAME=[docsgpt or openai or others] 
-   VITE_API_STREAMING=true
    API_KEY=[if LLM_NAME is openai]
    ```
 
-   See optional environment variables in the [/.env-template](https://github.com/arc53/DocsGPT/blob/main/.env-template) and [/application/.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) files.
+   See optional environment variables in the [/application/.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) file.
 
-3. Run [./run-with-docker-compose.sh](https://github.com/arc53/DocsGPT/blob/main/run-with-docker-compose.sh).
+3. Run the following command:
+
+   ```bash
+   docker compose up --build
+   ```
 4. Navigate to http://localhost:5173/.
 
 To stop, just run `Ctrl + C`.
 
-## Development Environments
-
-### Spin up Mongo and Redis
-
-For development, only two containers are used from [docker-compose.yaml](https://github.com/arc53/DocsGPT/blob/main/docker-compose.yaml) (by deleting all services except for Redis and Mongo).
-See file [docker-compose-dev.yaml](./docker-compose-dev.yaml).
-
-Run
-
-```
-docker compose -f docker-compose-dev.yaml build
-docker compose -f docker-compose-dev.yaml up -d
-```
-
-### Run the Backend
-
 > [!Note]
-> Make sure you have Python 3.10 or 3.11 installed.
-
-1. Export required environment variables or prepare a `.env` file in the project folder:
-   - Copy [.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) and create `.env`.
-
-(check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.)
-
-2. (optional) Create a Python virtual environment:
-   You can follow the [Python official documentation](https://docs.python.org/3/tutorial/venv.html) for virtual environments.
-
-a) On Mac OS and Linux
-
-```commandline
-python -m venv venv
-. venv/bin/activate
-```
-
-b) On Windows
-
-```commandline
-python -m venv venv
- venv/Scripts/activate
-```
-
-3. Download embedding model and save it in the `model/` folder:
-You can use the script below, or download it manually from [here](https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip), unzip it and save it in the `model/` folder.
-
-```commandline
-wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip
-unzip mpnet-base-v2.zip -d model
-rm mpnet-base-v2.zip
-```
-
-4. Install dependencies for the backend:
-
-```commandline
-pip install -r application/requirements.txt
-```
-
-5. Run the app using `flask --app application/app.py run --host=0.0.0.0 --port=7091`.
-6. Start worker with `celery -A application.app.celery worker -l INFO`.
-
-### Start Frontend
-
-> [!Note]
-> Make sure you have Node version 16 or higher.
-
-1. Navigate to the [/frontend](https://github.com/arc53/DocsGPT/tree/main/frontend) folder.
-2. Install the required packages `husky` and `vite` (ignore if already installed).
-
-```commandline
-npm install husky -g
-npm install vite -g
-```
-
-3. Install dependencies by running `npm install --include=dev`.
-4. Run the app using `npm run dev`.
+> For development environment setup instructions, please refer to the [Development Environment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).
 
 ## Contributing
 
 Please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file for information about how to get involved. We welcome issues, questions, and pull requests.
 
+## Architecture
+
+![Architecture chart](https://github.com/user-attachments/assets/fc6a7841-ddfc-45e6-b5a0-d05fe648cbe2)
+
+## Project Structure
+
+- Application - Flask app (main application).
+
+- Extensions - Extensions, like react widget or discord bot.
+
+- Frontend - Frontend uses <a href="https://vitejs.dev/">Vite</a> and <a href="https://react.dev/">React</a>.
+
+- Scripts - Miscellaneous scripts.
+
 ## Code Of Conduct
 
 We as members, contributors, and leaders, pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. Please refer to the [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) file for more information about contributing.
 
+
 ## Many Thanks To Our Contributors⚡
 
 <a href="https://github.com/arc53/DocsGPT/graphs/contributors" alt="View Contributors">
@@ -197,4 +144,9 @@ We as members, contributors, and leaders, pledge to make participation in our co
 
 The source code license is [MIT](https://opensource.org/license/mit/), as described in the [LICENSE](LICENSE) file.
 
-Built with [:bird: :link: LangChain](https://github.com/hwchase17/langchain)
+<p>This project is supported by:</p>
+<p>
+  <a href="https://www.digitalocean.com/?utm_medium=opensource&utm_source=DocsGPT">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px">
+  </a>
+</p>

application/Dockerfile

@@ -4,21 +4,18 @@ FROM ubuntu:24.04 as builder
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
-    apt-get install -y software-properties-common
-
-RUN add-apt-repository ppa:deadsnakes/ppa
-
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
 # Install necessary packages and Python
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends gcc curl wget unzip libc6-dev python3.11 python3.11-distutils python3.11-venv && \
-    apt-get clean && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends gcc wget unzip libc6-dev python3.12 python3.12-venv && \
     rm -rf /var/lib/apt/lists/* 
 
 # Verify Python installation and setup symlink
-RUN if [ -f /usr/bin/python3.11 ]; then \
-        ln -s /usr/bin/python3.11 /usr/bin/python; \
+RUN if [ -f /usr/bin/python3.12 ]; then \
+        ln -s /usr/bin/python3.12 /usr/bin/python; \
     else \
-        echo "Python 3.11 not found"; exit 1; \
+        echo "Python 3.12 not found"; exit 1; \
     fi
 
 # Download and unzip the model
@@ -27,7 +24,7 @@ RUN wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.z
     rm mpnet-base-v2.zip
 
 # Install Rust
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y
 
 # Clean up to reduce container size
 RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
@@ -36,7 +33,7 @@ RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var
 COPY requirements.txt .
 
 # Setup Python virtual environment
-RUN python3.11 -m venv /venv
+RUN python3.12 -m venv /venv
 
 # Activate virtual environment and install Python packages
 ENV PATH="/venv/bin:$PATH"
@@ -50,13 +47,11 @@ RUN pip install --no-cache-dir --upgrade pip && \
 FROM ubuntu:24.04 as final
 
 RUN apt-get update && \
-    apt-get install -y software-properties-common
-
-RUN add-apt-repository ppa:deadsnakes/ppa
-
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
 # Install Python
-RUN apt-get update && apt-get install -y --no-install-recommends python3.11 && \
-    ln -s /usr/bin/python3.11 /usr/bin/python && \
+    apt-get update && apt-get install -y --no-install-recommends python3.12 && \
+    ln -s /usr/bin/python3.12 /usr/bin/python && \
     rm -rf /var/lib/apt/lists/*
 
 # Set working directory

application/api/answer/routes.py

@@ -1,36 +1,47 @@
 import asyncio
+import datetime
+import json
+import logging
 import os
 import sys
-from flask import Blueprint, request, Response
-import json
-import datetime
-import logging
 import traceback
 
-from pymongo import MongoClient
+from bson.dbref import DBRef
 from bson.objectid import ObjectId
+from flask import Blueprint, current_app, make_response, request, Response
+from flask_restx import fields, Namespace, Resource
 
+
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
+from application.error import bad_request
+from application.extensions import api
 from application.llm.llm_creator import LLMCreator
 from application.retriever.retriever_creator import RetrieverCreator
-from application.error import bad_request
+from application.utils import check_required_fields, limit_chat_history
 
 logger = logging.getLogger(__name__)
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 prompts_collection = db["prompts"]
 api_key_collection = db["api_keys"]
+user_logs_collection = db["user_logs"]
+
 answer = Blueprint("answer", __name__)
+answer_ns = Namespace("answer", description="Answer related operations", path="/")
+api.add_namespace(answer_ns)
 
 gpt_model = ""
 # to have some kind of default behaviour
 if settings.LLM_NAME == "openai":
-    gpt_model = "gpt-3.5-turbo"
+    gpt_model = "gpt-4o-mini"
 elif settings.LLM_NAME == "anthropic":
     gpt_model = "claude-2"
+elif settings.LLM_NAME == "groq":
+    gpt_model = "llama3-8b-8192"
 
 if settings.MODEL_NAME:  # in case there is particular model name configured
     gpt_model = settings.MODEL_NAME
@@ -74,27 +85,29 @@ def run_async_chain(chain, question, chat_history):
 
 def get_data_from_api_key(api_key):
     data = api_key_collection.find_one({"key": api_key})
-    
     # # Raise custom exception if the API key is not found
     if data is None:
         raise Exception("Invalid API Key, please generate new key", 401)
+
+    if "retriever" not in data:
+        data["retriever"] = None
+
+    if "source" in data and isinstance(data["source"], DBRef):
+        source_doc = db.dereference(data["source"])
+        data["source"] = str(source_doc["_id"])
+        if "retriever" in source_doc:
+            data["retriever"] = source_doc["retriever"]
+    else:
+        data["source"] = {}
     return data
 
 
-def get_vectorstore(data):
-    if "active_docs" in data:
-        if data["active_docs"].split("/")[0] == "default":
-            vectorstore = ""
-        elif data["active_docs"].split("/")[0] == "local":
-            vectorstore = "indexes/" + data["active_docs"]
-        else:
-            vectorstore = "vectors/" + data["active_docs"]
-        if data["active_docs"] == "default":
-            vectorstore = ""
-    else:
-        vectorstore = ""
-    vectorstore = os.path.join("application", vectorstore)
-    return vectorstore
+def get_retriever(source_id: str):
+    doc = sources_collection.find_one({"_id": ObjectId(source_id)})
+    if doc is None:
+        raise Exception("Source document does not exist", 404)
+    retriever_name = None if "retriever" not in doc else doc["retriever"]
+    return retriever_name
 
 
 def is_azure_configured():
@@ -105,8 +118,31 @@ def is_azure_configured():
     )
 
 
-def save_conversation(conversation_id, question, response, source_log_docs, llm):
-    if conversation_id is not None and conversation_id != "None":
+def save_conversation(conversation_id, question, response, source_log_docs, llm,index=None):
+    if conversation_id is not None and index is not None:
+        conversations_collection.update_one(
+            {"_id": ObjectId(conversation_id), f"queries.{index}": {"$exists": True}},
+            {
+                "$set": {
+                    f"queries.{index}.prompt": question,
+                    f"queries.{index}.response": response,
+                    f"queries.{index}.sources": source_log_docs,
+                }
+            }
+        )
+        ##remove following queries from the array
+        conversations_collection.update_one(
+            {"_id": ObjectId(conversation_id), f"queries.{index}": {"$exists": True}},
+            {
+                "$push":{
+                    "queries":{
+                        "$each":[],
+                        "$slice":index+1
+                    }
+                }
+            }
+        )
+    elif conversation_id is not None and conversation_id != "None":
         conversations_collection.update_one(
             {"_id": ObjectId(conversation_id)},
             {
@@ -128,17 +164,17 @@ def save_conversation(conversation_id, question, response, source_log_docs, llm)
                 "role": "assistant",
                 "content": "Summarise following conversation in no more than 3 "
                 "words, respond ONLY with the summary, use the same "
-                "language as the system \n\nUser: "
-                +question
-                +"\n\n"
-                +"AI: "
-                +response,
+                "language as the system",
             },
             {
                 "role": "user",
                 "content": "Summarise following conversation in no more than 3 words, "
                 "respond ONLY with the summary, use the same language as the "
-                "system",
+                "system \n\nUser: "
+                + question
+                + "\n\n"
+                + "AI: "
+                + response,
             },
         ]
 
@@ -172,12 +208,21 @@ def get_prompt(prompt_id):
     return prompt
 
 
-def complete_stream(question, retriever, conversation_id, user_api_key):
+def complete_stream(
+    question, retriever, conversation_id, user_api_key, isNoneDoc=False,index=None
+):
 
     try:
         response_full = ""
         source_log_docs = []
         answer = retriever.gen()
+        sources = retriever.search()
+        for source in sources:
+            if "text" in source:
+                source["text"] = source["text"][:100].strip() + "..."
+        if len(sources) > 0:
+            data = json.dumps({"type": "source", "source": sources})
+            yield f"data: {data}\n\n"
         for line in answer:
             if "answer" in line:
                 response_full += str(line["answer"])
@@ -186,254 +231,416 @@ def complete_stream(question, retriever, conversation_id, user_api_key):
             elif "source" in line:
                 source_log_docs.append(line["source"])
 
+        if isNoneDoc:
+            for doc in source_log_docs:
+                doc["source"] = "None"
+
         llm = LLMCreator.create_llm(
             settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
-            )
-        conversation_id = save_conversation(
-            conversation_id, question, response_full, source_log_docs, llm
         )
-        
-        # send data.type = "end" to indicate that the stream has ended as json
-        data = json.dumps({"type": "id", "id": str(conversation_id)})
-        yield f"data: {data}\n\n"
+        if user_api_key is None:
+            conversation_id = save_conversation(
+                conversation_id, question, response_full, source_log_docs, llm,index
+            )
+            # send data.type = "end" to indicate that the stream has ended as json
+            data = json.dumps({"type": "id", "id": str(conversation_id)})
+            yield f"data: {data}\n\n"
+
+        retriever_params = retriever.get_params()
+        user_logs_collection.insert_one(
+            {
+                "action": "stream_answer",
+                "level": "info",
+                "user": "local",
+                "api_key": user_api_key,
+                "question": question,
+                "response": response_full,
+                "sources": source_log_docs,
+                "retriever_params": retriever_params,
+                "timestamp": datetime.datetime.now(datetime.timezone.utc),
+            }
+        )
         data = json.dumps({"type": "end"})
         yield f"data: {data}\n\n"
     except Exception as e:
         print("\033[91merr", str(e), file=sys.stderr)
-        data = json.dumps({"type": "error","error":"Please try again later. We apologize for any inconvenience.",
-          "error_exception": str(e)})
-        yield f"data: {data}\n\n"
-        return 
-
-@answer.route("/stream", methods=["POST"])
-def stream():
-   try:
-    data = request.get_json()
-    # get parameter from url question
-    question = data["question"]
-    if "history" not in data:
-        history = []
-    else:
-        history = data["history"]
-        history = json.loads(history)
-    if "conversation_id" not in data:
-        conversation_id = None
-    else:
-        conversation_id = data["conversation_id"]
-    if "prompt_id" in data:
-        prompt_id = data["prompt_id"]
-    else:
-        prompt_id = "default"
-    if "selectedDocs" in data and data["selectedDocs"] is None:
-        chunks = 0
-    elif "chunks" in data:
-        chunks = int(data["chunks"])
-    else:
-        chunks = 2
-    if "token_limit" in data:
-        token_limit = data["token_limit"]
-    else:
-        token_limit = settings.DEFAULT_MAX_HISTORY
-
-    # check if active_docs or api_key is set
-
-    if "api_key" in data:
-        data_key = get_data_from_api_key(data["api_key"])
-        chunks = int(data_key["chunks"])
-        prompt_id = data_key["prompt_id"]
-        source = {"active_docs": data_key["source"]}
-        user_api_key = data["api_key"]
-    elif "active_docs" in data:
-        source = {"active_docs": data["active_docs"]}
-        user_api_key = None
-    else:
-        source = {}
-        user_api_key = None
-
-    if (
-        source["active_docs"].split("/")[0] == "default"
-        or source["active_docs"].split("/")[0] == "local"
-    ):
-        retriever_name = "classic"
-    else:
-        retriever_name = source["active_docs"]
-
-    prompt = get_prompt(prompt_id)
-
-    retriever = RetrieverCreator.create_retriever(
-        retriever_name,
-        question=question,
-        source=source,
-        chat_history=history,
-        prompt=prompt,
-        chunks=chunks,
-        token_limit=token_limit,
-        gpt_model=gpt_model,
-        user_api_key=user_api_key,
-    )
-
-    return Response(
-        complete_stream(
-            question=question,
-            retriever=retriever,
-            conversation_id=conversation_id,
-            user_api_key=user_api_key,
-        ),
-        mimetype="text/event-stream",
-    )
-    
-   except ValueError:
-       message = "Malformed request body"
-       print("\033[91merr", str(message), file=sys.stderr)
-       return Response(
-        error_stream_generate(message),
-        status=400,
-        mimetype="text/event-stream",
-    )
-   except Exception as e:
-        print("\033[91merr", str(e), file=sys.stderr)
-        message = e.args[0]
-        status_code = 400
-        # # Custom exceptions with two arguments, index 1 as status code
-        if(len(e.args) >= 2):
-            status_code = e.args[1]
-        return Response(
-        error_stream_generate(message),
-        status=status_code,
-        mimetype="text/event-stream",
-    )
-def error_stream_generate(err_response):
-            data = json.dumps({"type": "error", "error":err_response})
-            yield f"data: {data}\n\n"
-
-@answer.route("/api/answer", methods=["POST"])
-def api_answer():
-    data = request.get_json()
-    question = data["question"]
-    if "history" not in data:
-        history = []
-    else:
-        history = data["history"]
-    if "conversation_id" not in data:
-        conversation_id = None
-    else:
-        conversation_id = data["conversation_id"]
-    print("-" * 5)
-    if "prompt_id" in data:
-        prompt_id = data["prompt_id"]
-    else:
-        prompt_id = "default"
-    if "chunks" in data:
-        chunks = int(data["chunks"])
-    else:
-        chunks = 2
-    if "token_limit" in data:
-        token_limit = data["token_limit"]
-    else:
-        token_limit = settings.DEFAULT_MAX_HISTORY
-
-    # use try and except  to check for exception
-    try:
-        # check if the vectorstore is set
-        if "api_key" in data:
-            data_key = get_data_from_api_key(data["api_key"])
-            chunks = int(data_key["chunks"])
-            prompt_id = data_key["prompt_id"]
-            source = {"active_docs": data_key["source"]}
-            user_api_key = data["api_key"]
-        else:
-            source = data
-            user_api_key = None
-
-        if (
-            source["active_docs"].split("/")[0] == "default"
-            or source["active_docs"].split("/")[0] == "local"
-        ):
-            retriever_name = "classic"
-        else:
-            retriever_name = source["active_docs"]
-
-        prompt = get_prompt(prompt_id)
-
-        retriever = RetrieverCreator.create_retriever(
-            retriever_name,
-            question=question,
-            source=source,
-            chat_history=history,
-            prompt=prompt,
-            chunks=chunks,
-            token_limit=token_limit,
-            gpt_model=gpt_model,
-            user_api_key=user_api_key,
-        )
-        source_log_docs = []
-        response_full = ""
-        for line in retriever.gen():
-            if "source" in line:
-                source_log_docs.append(line["source"])
-            elif "answer" in line:
-                response_full += line["answer"]
-
-        llm = LLMCreator.create_llm(
-            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
-        )
-
-        result = {"answer": response_full, "sources": source_log_docs}
-        result["conversation_id"] = save_conversation(
-            conversation_id, question, response_full, source_log_docs, llm
-        )
-
-        return result
-    except Exception as e:
-        # print whole traceback
         traceback.print_exc()
-        print(str(e))
-        return bad_request(500, str(e))
+        data = json.dumps(
+            {
+                "type": "error",
+                "error": "Please try again later. We apologize for any inconvenience.",
+                "error_exception": str(e),
+            }
+        )
+        yield f"data: {data}\n\n"
+        return
 
 
-@answer.route("/api/search", methods=["POST"])
-def api_search():
-    data = request.get_json()
-    # get parameter from url question
-    question = data["question"]
-    if "chunks" in data:
-        chunks = int(data["chunks"])
-    else:
-        chunks = 2
-    if "api_key" in data:
-        data_key = get_data_from_api_key(data["api_key"])
-        chunks = int(data_key["chunks"])
-        source = {"active_docs": data_key["source"]}
-        user_api_key = data["api_key"]
-    elif "active_docs" in data:
-        source = {"active_docs": data["active_docs"]}
-        user_api_key = None
-    else:
-        source = {}
-        user_api_key = None
-
-    if (
-        source["active_docs"].split("/")[0] == "default"
-        or source["active_docs"].split("/")[0] == "local"
-    ):
-        retriever_name = "classic"
-    else:
-        retriever_name = source["active_docs"]
-    if "token_limit" in data:
-        token_limit = data["token_limit"]
-    else:
-        token_limit = settings.DEFAULT_MAX_HISTORY
-
-    retriever = RetrieverCreator.create_retriever(
-        retriever_name,
-        question=question,
-        source=source,
-        chat_history=[],
-        prompt="default",
-        chunks=chunks,
-        token_limit=token_limit,
-        gpt_model=gpt_model,
-        user_api_key=user_api_key,
+@answer_ns.route("/stream")
+class Stream(Resource):
+    stream_model = api.model(
+        "StreamModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
+            ),
+            "history": fields.List(
+                fields.String, required=False, description="Chat history"
+            ),
+            "conversation_id": fields.String(
+                required=False, description="Conversation ID"
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+            "index":fields.Integer(
+                required=False, description="The position where query is to be updated"
+            ),
+        },
     )
-    docs = retriever.search()
-    return docs
+
+    @api.expect(stream_model)
+    @api.doc(description="Stream a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]
+        if "index" in data:
+            required_fields = ["question","conversation_id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            history = limit_chat_history(json.loads(data.get("history", [])), gpt_model=gpt_model)
+            conversation_id = data.get("conversation_id")
+            prompt_id = data.get("prompt_id", "default")
+            
+            index=data.get("index",None)
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+            
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                prompt_id = data_key.get("prompt_id", "default")
+                source = {"active_docs": data_key.get("source")}
+                retriever_name = data_key.get("retriever", retriever_name)
+                user_api_key = data["api_key"]
+
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                retriever_name = get_retriever(data["active_docs"]) or retriever_name
+                user_api_key = None
+
+            else:
+                source = {}
+                user_api_key = None
+
+            current_app.logger.info(
+                f"/stream - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            prompt = get_prompt(prompt_id)
+            if "isNoneDoc" in data and data["isNoneDoc"] is True:
+                chunks = 0
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=history,
+                prompt=prompt,
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+            
+            return Response(
+                complete_stream(
+                    question=question,
+                    retriever=retriever,
+                    conversation_id=conversation_id,
+                    user_api_key=user_api_key,
+                    isNoneDoc=data.get("isNoneDoc"),
+                    index=index,
+                ),
+                mimetype="text/event-stream",
+            )
+
+        except ValueError:
+            message = "Malformed request body"
+            print("\033[91merr", str(message), file=sys.stderr)
+            return Response(
+                error_stream_generate(message),
+                status=400,
+                mimetype="text/event-stream",
+            )
+        except Exception as e:
+            current_app.logger.error(
+                f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            message = e.args[0]
+            status_code = 400
+            # Custom exceptions with two arguments, index 1 as status code
+            if len(e.args) >= 2:
+                status_code = e.args[1]
+            return Response(
+                error_stream_generate(message),
+                status=status_code,
+                mimetype="text/event-stream",
+            )
+
+
+def error_stream_generate(err_response):
+    data = json.dumps({"type": "error", "error": err_response})
+    yield f"data: {data}\n\n"
+
+
+@answer_ns.route("/api/answer")
+class Answer(Resource):
+    answer_model = api.model(
+        "AnswerModel",
+        {
+            "question": fields.String(
+                required=True, description="The question to answer"
+            ),
+            "history": fields.List(
+                fields.String, required=False, description="Conversation history"
+            ),
+            "conversation_id": fields.String(
+                required=False, description="Conversation ID"
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
+    )
+
+    @api.expect(answer_model)
+    @api.doc(description="Provide an answer based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]       
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            history = limit_chat_history(json.loads(data.get("history", [])), gpt_model=gpt_model)
+            conversation_id = data.get("conversation_id")
+            prompt_id = data.get("prompt_id", "default")
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                prompt_id = data_key.get("prompt_id", "default")
+                source = {"active_docs": data_key.get("source")}
+                retriever_name = data_key.get("retriever", retriever_name)
+                user_api_key = data["api_key"]
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                retriever_name = get_retriever(data["active_docs"]) or retriever_name
+                user_api_key = None
+            else:
+                source = {}
+                user_api_key = None
+
+            prompt = get_prompt(prompt_id)
+
+            current_app.logger.info(
+                f"/api/answer - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=history,
+                prompt=prompt,
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            source_log_docs = []
+            response_full = ""
+            for line in retriever.gen():
+                if "source" in line:
+                    source_log_docs.append(line["source"])
+                elif "answer" in line:
+                    response_full += line["answer"]
+
+            if data.get("isNoneDoc"):
+                for doc in source_log_docs:
+                    doc["source"] = "None"
+
+            llm = LLMCreator.create_llm(
+                settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key
+            )
+
+            result = {"answer": response_full, "sources": source_log_docs}
+            result["conversation_id"] = str(
+                save_conversation(
+                    conversation_id, question, response_full, source_log_docs, llm
+                )
+            )
+            retriever_params = retriever.get_params()
+            user_logs_collection.insert_one(
+                {
+                    "action": "api_answer",
+                    "level": "info",
+                    "user": "local",
+                    "api_key": user_api_key,
+                    "question": question,
+                    "response": response_full,
+                    "sources": source_log_docs,
+                    "retriever_params": retriever_params,
+                    "timestamp": datetime.datetime.now(datetime.timezone.utc),
+                }
+            )
+
+        except Exception as e:
+            current_app.logger.error(
+                f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return bad_request(500, str(e))
+
+        return make_response(result, 200)
+
+
+@answer_ns.route("/api/search")
+class Search(Resource):
+    search_model = api.model(
+        "SearchModel",
+        {
+            "question": fields.String(
+                required=True, description="The question to search"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "api_key": fields.String(
+                required=False, description="API key for authentication"
+            ),
+            "active_docs": fields.String(
+                required=False, description="Active documents for retrieval"
+            ),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "token_limit": fields.Integer(
+                required=False, description="Limit for tokens"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+        },
+    )
+
+    @api.expect(search_model)
+    @api.doc(
+        description="Search for relevant documents based on the question and retriever"
+    )
+    def post(self):
+        data = request.get_json()
+        required_fields = ["question"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            question = data["question"]
+            chunks = int(data.get("chunks", 2))
+            token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
+            retriever_name = data.get("retriever", "classic")
+
+            if "api_key" in data:
+                data_key = get_data_from_api_key(data["api_key"])
+                chunks = int(data_key.get("chunks", 2))
+                source = {"active_docs": data_key.get("source")}
+                user_api_key = data["api_key"]
+            elif "active_docs" in data:
+                source = {"active_docs": data["active_docs"]}
+                user_api_key = None
+            else:
+                source = {}
+                user_api_key = None
+
+            current_app.logger.info(
+                f"/api/answer - request_data: {data}, source: {source}",
+                extra={"data": json.dumps({"request_data": data, "source": source})},
+            )
+
+            retriever = RetrieverCreator.create_retriever(
+                retriever_name,
+                question=question,
+                source=source,
+                chat_history=[],
+                prompt="default",
+                chunks=chunks,
+                token_limit=token_limit,
+                gpt_model=gpt_model,
+                user_api_key=user_api_key,
+            )
+
+            docs = retriever.search()
+            retriever_params = retriever.get_params()
+
+            user_logs_collection.insert_one(
+                {
+                    "action": "api_search",
+                    "level": "info",
+                    "user": "local",
+                    "api_key": user_api_key,
+                    "question": question,
+                    "sources": docs,
+                    "retriever_params": retriever_params,
+                    "timestamp": datetime.datetime.now(datetime.timezone.utc),
+                }
+            )
+
+            if data.get("isNoneDoc"):
+                for doc in docs:
+                    doc["source"] = "None"
+
+        except Exception as e:
+            current_app.logger.error(
+                f"/api/search - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return bad_request(500, str(e))
+
+        return make_response(docs, 200)

application/api/internal/routes.py

@@ -1,20 +1,25 @@
 import os
 import datetime
 from flask import Blueprint, request, send_from_directory
-from pymongo import MongoClient
 from werkzeug.utils import secure_filename
+from bson.objectid import ObjectId
 
-
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
-mongo = MongoClient(settings.MONGO_URI)
+
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+
+internal = Blueprint("internal", __name__)
 
 
-internal = Blueprint('internal', __name__)
 @internal.route("/api/download", methods=["get"])
 def download_file():
     user = secure_filename(request.args.get("user"))
@@ -24,7 +29,6 @@ def download_file():
     return send_from_directory(save_dir, filename, as_attachment=True)
 
 
-
 @internal.route("/api/upload_index", methods=["POST"])
 def upload_index_files():
     """Upload two files(index.faiss, index.pkl) to the user's folder."""
@@ -35,7 +39,13 @@ def upload_index_files():
         return {"status": "no name"}
     job_name = secure_filename(request.form["name"])
     tokens = secure_filename(request.form["tokens"])
-    save_dir = os.path.join(current_dir, "indexes", user, job_name)
+    retriever = secure_filename(request.form["retriever"])
+    id = secure_filename(request.form["id"])
+    type = secure_filename(request.form["type"])
+    remote_data = request.form["remote_data"] if "remote_data" in request.form else None
+    sync_frequency = secure_filename(request.form["sync_frequency"]) if "sync_frequency" in request.form else None
+
+    save_dir = os.path.join(current_dir, "indexes", str(id))
     if settings.VECTOR_STORE == "faiss":
         if "file_faiss" not in request.files:
             print("No file part")
@@ -50,22 +60,45 @@ def upload_index_files():
         if file_pkl.filename == "":
             return {"status": "no file name"}
         # saves index files
-        
+
         if not os.path.exists(save_dir):
             os.makedirs(save_dir)
         file_faiss.save(os.path.join(save_dir, "index.faiss"))
         file_pkl.save(os.path.join(save_dir, "index.pkl"))
-    # create entry in vectors_collection
-    vectors_collection.insert_one(
-        {
-            "user": user,
-            "name": job_name,
-            "language": job_name,
-            "location": save_dir,
-            "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
-            "model": settings.EMBEDDINGS_NAME,
-            "type": "local",
-            "tokens": tokens
-        }
-    )
-    return {"status": "ok"}
+
+    existing_entry = sources_collection.find_one({"_id": ObjectId(id)})
+    if existing_entry:
+        sources_collection.update_one(
+            {"_id": ObjectId(id)},
+            {
+                "$set": {
+                    "user": user,
+                    "name": job_name,
+                    "language": job_name,
+                    "date": datetime.datetime.now(),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "type": type,
+                    "tokens": tokens,
+                    "retriever": retriever,
+                    "remote_data": remote_data,
+                    "sync_frequency": sync_frequency,
+                }
+            },
+        )
+    else:
+        sources_collection.insert_one(
+            {
+                "_id": ObjectId(id),
+                "user": user,
+                "name": job_name,
+                "language": job_name,
+                "date": datetime.datetime.now(),
+                "model": settings.EMBEDDINGS_NAME,
+                "type": type,
+                "tokens": tokens,
+                "retriever": retriever,
+                "remote_data": remote_data,
+                "sync_frequency": sync_frequency,
+            }
+        )
+    return {"status": "ok"}

application/api/user/tasks.py

@@ -1,12 +1,38 @@
-from application.worker import ingest_worker, remote_worker
+from datetime import timedelta
+
 from application.celery_init import celery
+from application.worker import ingest_worker, remote_worker, sync_worker
+
 
 @celery.task(bind=True)
 def ingest(self, directory, formats, name_job, filename, user):
     resp = ingest_worker(self, directory, formats, name_job, filename, user)
     return resp
 
+
 @celery.task(bind=True)
 def ingest_remote(self, source_data, job_name, user, loader):
     resp = remote_worker(self, source_data, job_name, user, loader)
     return resp
+
+
+@celery.task(bind=True)
+def schedule_syncs(self, frequency):
+    resp = sync_worker(self, frequency)
+    return resp
+
+
+@celery.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+    sender.add_periodic_task(
+        timedelta(days=1),
+        schedule_syncs.s("daily"),
+    )
+    sender.add_periodic_task(
+        timedelta(weeks=1),
+        schedule_syncs.s("weekly"),
+    )
+    sender.add_periodic_task(
+        timedelta(days=30),
+        schedule_syncs.s("monthly"),
+    )

application/app.py

@@ -1,17 +1,23 @@
 import platform
+
 import dotenv
-from application.celery_init import celery
-from flask import Flask, request, redirect
-from application.core.settings import settings
-from application.api.user.routes import user
+from flask import Flask, redirect, request
+
 from application.api.answer.routes import answer
 from application.api.internal.routes import internal
+from application.api.user.routes import user
+from application.celery_init import celery
+from application.core.logging_config import setup_logging
+from application.core.settings import settings
+from application.extensions import api
 
 if platform.system() == "Windows":
     import pathlib
+
     pathlib.PosixPath = pathlib.WindowsPath
 
 dotenv.load_dotenv()
+setup_logging()
 
 app = Flask(__name__)
 app.register_blueprint(user)
@@ -21,16 +27,19 @@ app.config.update(
     UPLOAD_FOLDER="inputs",
     CELERY_BROKER_URL=settings.CELERY_BROKER_URL,
     CELERY_RESULT_BACKEND=settings.CELERY_RESULT_BACKEND,
-    MONGO_URI=settings.MONGO_URI
+    MONGO_URI=settings.MONGO_URI,
 )
 celery.config_from_object("application.celeryconfig")
+api.init_app(app)
+
 
 @app.route("/")
 def home():
-    if request.remote_addr in ('0.0.0.0', '127.0.0.1', 'localhost', '172.18.0.1'):
-        return redirect('http://localhost:5173')
+    if request.remote_addr in ("0.0.0.0", "127.0.0.1", "localhost", "172.18.0.1"):
+        return redirect("http://localhost:5173")
     else:
-        return 'Welcome to DocsGPT Backend!'
+        return "Welcome to DocsGPT Backend!"
+
 
 @app.after_request
 def after_request(response):
@@ -39,6 +48,6 @@ def after_request(response):
     response.headers.add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS")
     return response
 
+
 if __name__ == "__main__":
     app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)
-

application/cache.py

@@ -0,0 +1,103 @@
+import json
+import logging
+import time
+from threading import Lock
+
+import redis
+
+from application.core.settings import settings
+from application.utils import get_hash
+
+logger = logging.getLogger(__name__)
+
+_redis_instance = None
+_instance_lock = Lock()
+
+
+def get_redis_instance():
+    global _redis_instance
+    if _redis_instance is None:
+        with _instance_lock:
+            if _redis_instance is None:
+                try:
+                    _redis_instance = redis.Redis.from_url(
+                        settings.CACHE_REDIS_URL, socket_connect_timeout=2
+                    )
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+                    _redis_instance = None
+    return _redis_instance
+
+
+def gen_cache_key(messages, model="docgpt", tools=None):
+    if not all(isinstance(msg, dict) for msg in messages):
+        raise ValueError("All messages must be dictionaries.")
+    messages_str = json.dumps(messages)
+    tools_str = json.dumps(str(tools)) if tools else ""
+    combined = f"{model}_{messages_str}_{tools_str}"
+    cache_key = get_hash(combined)
+    return cache_key
+
+
+def gen_cache(func):
+    def wrapper(self, model, messages, stream, tools=None, *args, **kwargs):
+        try:
+            cache_key = gen_cache_key(messages, model, tools)
+            redis_client = get_redis_instance()
+            if redis_client:
+                try:
+                    cached_response = redis_client.get(cache_key)
+                    if cached_response:
+                        return cached_response.decode("utf-8")
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            result = func(self, model, messages, stream, tools, *args, **kwargs)
+            if redis_client and isinstance(result, str):
+                try:
+                    redis_client.set(cache_key, result, ex=1800)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            return result
+        except ValueError as e:
+            logger.error(e)
+            return "Error: No user message found in the conversation to generate a cache key."
+
+    return wrapper
+
+
+def stream_cache(func):
+    def wrapper(self, model, messages, stream, tools=None, *args, **kwargs):
+        cache_key = gen_cache_key(messages, model, tools)
+        logger.info(f"Stream cache key: {cache_key}")
+
+        redis_client = get_redis_instance()
+        if redis_client:
+            try:
+                cached_response = redis_client.get(cache_key)
+                if cached_response:
+                    logger.info(f"Cache hit for stream key: {cache_key}")
+                    cached_response = json.loads(cached_response.decode("utf-8"))
+                    for chunk in cached_response:
+                        yield chunk
+                        time.sleep(0.03)
+                    return
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+
+        result = func(self, model, messages, stream, tools=tools, *args, **kwargs)
+        stream_cache_data = []
+
+        for chunk in result:
+            stream_cache_data.append(chunk)
+            yield chunk
+
+        if redis_client:
+            try:
+                redis_client.set(cache_key, json.dumps(stream_cache_data), ex=1800)
+                logger.info(f"Stream cache saved for key: {cache_key}")
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+
+    return wrapper

application/celery_init.py

@@ -1,9 +1,23 @@
 from celery import Celery
 from application.core.settings import settings
+from celery.signals import setup_logging
+
 
 def make_celery(app_name=__name__):
-    celery = Celery(app_name, broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND)
+    celery = Celery(
+        app_name,
+        broker=settings.CELERY_BROKER_URL,
+        backend=settings.CELERY_RESULT_BACKEND,
+    )
     celery.conf.update(settings)
     return celery
 
+
+@setup_logging.connect
+def config_loggers(*args, **kwargs):
+    from application.core.logging_config import setup_logging
+
+    setup_logging()
+
+
 celery = make_celery()

application/core/logging_config.py

@@ -0,0 +1,22 @@
+from logging.config import dictConfig
+
+def setup_logging():
+    dictConfig({
+        'version': 1,
+        'formatters': {
+            'default': {
+                'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+            }
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "stream": "ext://sys.stdout",
+                "formatter": "default",
+            }
+        },
+        'root': {
+            'level': 'INFO',
+            'handlers': ['console'],
+        },
+    })

application/core/mongo_db.py

@@ -0,0 +1,24 @@
+from application.core.settings import settings
+from pymongo import MongoClient
+
+
+class MongoDB:
+    _client = None
+
+    @classmethod
+    def get_client(cls):
+        """
+        Get the MongoDB client instance, creating it if necessary.
+        """
+        if cls._client is None:
+            cls._client = MongoClient(settings.MONGO_URI)
+        return cls._client
+
+    @classmethod
+    def close_client(cls):
+        """
+        Close the MongoDB client connection.
+        """
+        if cls._client is not None:
+            cls._client.close()
+            cls._client = None

application/core/settings.py

@@ -16,11 +16,15 @@ class Settings(BaseSettings):
     MONGO_URI: str = "mongodb://localhost:27017/docsgpt"
     MODEL_PATH: str = os.path.join(current_dir, "models/docsgpt-7b-f16.gguf")
     DEFAULT_MAX_HISTORY: int = 150
-    MODEL_TOKEN_LIMITS: dict = {"gpt-3.5-turbo": 4096, "claude-2": 1e5}
+    MODEL_TOKEN_LIMITS: dict = {"gpt-4o-mini": 128000, "gpt-3.5-turbo": 4096, "claude-2": 1e5}
     UPLOAD_FOLDER: str = "inputs"
-    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant"
+    PARSE_PDF_AS_IMAGE: bool = False
+    VECTOR_STORE: str = "faiss" #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb"
     RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
 
+    # LLM Cache
+    CACHE_REDIS_URL: str = "redis://localhost:6379/2"
+
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
 
     API_KEY: Optional[str] = None  # LLM api key
@@ -29,6 +33,7 @@ class Settings(BaseSettings):
     OPENAI_API_VERSION: Optional[str] = None  # azure openai api version
     AZURE_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for answering
     AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for embeddings
+    OPENAI_BASE_URL: Optional[str] = None # openai base url for open ai compatable models
 
     # elasticsearch
     ELASTIC_CLOUD_ID: Optional[str] = None  # cloud id for elasticsearch
@@ -61,6 +66,14 @@ class Settings(BaseSettings):
     QDRANT_PATH: Optional[str] = None
     QDRANT_DISTANCE_FUNC: str = "Cosine"
 
+    # Milvus vectorstore config
+    MILVUS_COLLECTION_NAME: Optional[str] = "docsgpt"
+    MILVUS_URI: Optional[str] = "./milvus_local.db"   # milvus lite version as default
+    MILVUS_TOKEN: Optional[str] = ""
+
+    # LanceDB vectorstore config
+    LANCEDB_PATH: str = "/tmp/lancedb"  # Path where LanceDB stores its local data
+    LANCEDB_TABLE_NAME: Optional[str] = "docsgpts"  # Name of the table to use for storing vectors
     BRAVE_SEARCH_API_KEY: Optional[str] = None
 
     FLASK_DEBUG_MODE: bool = False

application/extensions.py

@@ -0,0 +1,7 @@
+from flask_restx import Api
+
+api = Api(
+    version="1.0",
+    title="DocsGPT API",
+    description="API for DocsGPT",
+)

application/llm/anthropic.py

@@ -17,7 +17,7 @@ class AnthropicLLM(BaseLLM):
         self.AI_PROMPT = AI_PROMPT
 
     def _raw_gen(
-        self, baseself, model, messages, stream=False, max_tokens=300, **kwargs
+        self, baseself, model, messages, stream=False, tools=None, max_tokens=300, **kwargs
     ):
         context = messages[0]["content"]
         user_question = messages[-1]["content"]
@@ -34,7 +34,7 @@ class AnthropicLLM(BaseLLM):
         return completion.completion
 
     def _raw_gen_stream(
-        self, baseself, model, messages, stream=True, max_tokens=300, **kwargs
+        self, baseself, model, messages, stream=True, tools=None, max_tokens=300, **kwargs
     ):
         context = messages[0]["content"]
         user_question = messages[-1]["content"]

application/llm/base.py

@@ -1,4 +1,6 @@
 from abc import ABC, abstractmethod
+
+from application.cache import gen_cache, stream_cache
 from application.usage import gen_token_usage, stream_token_usage
 
 
@@ -6,23 +8,49 @@ class BaseLLM(ABC):
     def __init__(self):
         self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
 
-    def _apply_decorator(self, method, decorator, *args, **kwargs):
-        return decorator(method, *args, **kwargs)
+    def _apply_decorator(self, method, decorators, *args, **kwargs):
+        for decorator in decorators:
+            method = decorator(method)
+        return method(self, *args, **kwargs)
 
     @abstractmethod
-    def _raw_gen(self, model, messages, stream, *args, **kwargs):
+    def _raw_gen(self, model, messages, stream, tools, *args, **kwargs):
         pass
 
-    def gen(self, model, messages, stream=False, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen, gen_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
+    def gen(self, model, messages, stream=False, tools=None, *args, **kwargs):
+        decorators = [gen_token_usage, gen_cache]
+        return self._apply_decorator(
+            self._raw_gen,
+            decorators=decorators,
+            model=model,
+            messages=messages,
+            stream=stream,
+            tools=tools,
+            *args,
+            **kwargs
         )
 
     @abstractmethod
     def _raw_gen_stream(self, model, messages, stream, *args, **kwargs):
         pass
 
-    def gen_stream(self, model, messages, stream=True, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen_stream, stream_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
+    def gen_stream(self, model, messages, stream=True, tools=None, *args, **kwargs):
+        decorators = [stream_cache, stream_token_usage]
+        return self._apply_decorator(
+            self._raw_gen_stream,
+            decorators=decorators,
+            model=model,
+            messages=messages,
+            stream=stream,
+            tools=tools,
+            *args,
+            **kwargs
         )
+
+    def supports_tools(self):
+        return hasattr(self, "_supports_tools") and callable(
+            getattr(self, "_supports_tools")
+        )
+
+    def _supports_tools(self):
+        raise NotImplementedError("Subclass must implement _supports_tools method")

application/llm/docsgpt_provider.py

@@ -9,35 +9,25 @@ class DocsGPTAPILLM(BaseLLM):
         super().__init__(*args, **kwargs)
         self.api_key = api_key
         self.user_api_key = user_api_key
-        self.endpoint = "https://llm.docsgpt.co.uk"
+        self.endpoint = "https://llm.arc53.com"
 
     def _raw_gen(self, baseself, model, messages, stream=False, *args, **kwargs):
-        context = messages[0]["content"]
-        user_question = messages[-1]["content"]
-        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
         response = requests.post(
-            f"{self.endpoint}/answer", json={"prompt": prompt, "max_new_tokens": 30}
+            f"{self.endpoint}/answer", json={"messages": messages, "max_new_tokens": 30}
         )
         response_clean = response.json()["a"].replace("###", "")
 
         return response_clean
 
     def _raw_gen_stream(self, baseself, model, messages, stream=True, *args, **kwargs):
-        context = messages[0]["content"]
-        user_question = messages[-1]["content"]
-        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
-        # send prompt to endpoint /stream
         response = requests.post(
             f"{self.endpoint}/stream",
-            json={"prompt": prompt, "max_new_tokens": 256},
+            json={"messages": messages, "max_new_tokens": 256},
             stream=True,
         )
 
         for line in response.iter_lines():
             if line:
-                # data = json.loads(line)
                 data_str = line.decode("utf-8")
                 if data_str.startswith("data: "):
                     data = json.loads(data_str[6:])

application/llm/google_ai.py

@@ -0,0 +1,159 @@
+from google import genai
+from google.genai import types
+
+from application.llm.base import BaseLLM
+
+
+class GoogleLLM(BaseLLM):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _clean_messages_google(self, messages):
+        cleaned_messages = []
+        for message in messages:
+            role = message.get("role")
+            content = message.get("content")
+
+            if role == "assistant":
+                role = "model"
+
+            parts = []
+            if role and content is not None:
+                if isinstance(content, str):
+                    parts = [types.Part.from_text(content)]
+                elif isinstance(content, list):
+                    for item in content:
+                        if "text" in item:
+                            parts.append(types.Part.from_text(item["text"]))
+                        elif "function_call" in item:
+                            parts.append(
+                                types.Part.from_function_call(
+                                    name=item["function_call"]["name"],
+                                    args=item["function_call"]["args"],
+                                )
+                            )
+                        elif "function_response" in item:
+                            parts.append(
+                                types.Part.from_function_response(
+                                    name=item["function_response"]["name"],
+                                    response=item["function_response"]["response"],
+                                )
+                            )
+                        else:
+                            raise ValueError(
+                                f"Unexpected content dictionary format:{item}"
+                            )
+                else:
+                    raise ValueError(f"Unexpected content type: {type(content)}")
+
+                cleaned_messages.append(types.Content(role=role, parts=parts))
+
+        return cleaned_messages
+
+    def _clean_tools_format(self, tools_list):
+        genai_tools = []
+        for tool_data in tools_list:
+            if tool_data["type"] == "function":
+                function = tool_data["function"]
+                parameters = function["parameters"]
+                properties = parameters.get("properties", {})
+
+                if properties:
+                    genai_function = dict(
+                        name=function["name"],
+                        description=function["description"],
+                        parameters={
+                            "type": "OBJECT",
+                            "properties": {
+                                k: {
+                                    **v,
+                                    "type": v["type"].upper() if v["type"] else None,
+                                }
+                                for k, v in properties.items()
+                            },
+                            "required": (
+                                parameters["required"]
+                                if "required" in parameters
+                                else []
+                            ),
+                        },
+                    )
+                else:
+                    genai_function = dict(
+                        name=function["name"],
+                        description=function["description"],
+                    )
+
+                genai_tool = types.Tool(function_declarations=[genai_function])
+                genai_tools.append(genai_tool)
+
+        return genai_tools
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        tools=None,
+        formatting="openai",
+        **kwargs,
+    ):
+        client = genai.Client(api_key=self.api_key)
+        if formatting == "openai":
+            messages = self._clean_messages_google(messages)
+        config = types.GenerateContentConfig()
+        if messages[0].role == "system":
+            config.system_instruction = messages[0].parts[0].text
+            messages = messages[1:]
+
+        if tools:
+            cleaned_tools = self._clean_tools_format(tools)
+            config.tools = cleaned_tools
+            response = client.models.generate_content(
+                model=model,
+                contents=messages,
+                config=config,
+            )
+            return response
+        else:
+            response = client.models.generate_content(
+                model=model, contents=messages, config=config
+            )
+            return response.text
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        tools=None,
+        formatting="openai",
+        **kwargs,
+    ):
+        client = genai.Client(api_key=self.api_key)
+        if formatting == "openai":
+            messages = self._clean_messages_google(messages)
+        config = types.GenerateContentConfig()
+        if messages[0].role == "system":
+            config.system_instruction = messages[0].parts[0].text
+            messages = messages[1:]
+
+        if tools:
+            cleaned_tools = self._clean_tools_format(tools)
+            config.tools = cleaned_tools
+
+        response = client.models.generate_content_stream(
+            model=model,
+            contents=messages,
+            config=config,
+        )
+        for chunk in response:
+            if chunk.text is not None:
+                yield chunk.text
+
+    def _supports_tools(self):
+        return True

application/llm/groq.py

@@ -0,0 +1,32 @@
+from application.llm.base import BaseLLM
+from openai import OpenAI
+
+
+class GroqLLM(BaseLLM):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs):
+        if tools:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, tools=tools, **kwargs
+            )
+            return response.choices[0]
+        else:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, **kwargs
+            )
+            return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self, baseself, model, messages, stream=True, tools=None, **kwargs
+    ):
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+        for line in response:
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content

application/llm/llm_creator.py

@@ -1,3 +1,4 @@
+from application.llm.groq import GroqLLM
 from application.llm.openai import OpenAILLM, AzureOpenAILLM
 from application.llm.sagemaker import SagemakerAPILLM
 from application.llm.huggingface import HuggingFaceLLM
@@ -5,6 +6,7 @@ from application.llm.llama_cpp import LlamaCpp
 from application.llm.anthropic import AnthropicLLM
 from application.llm.docsgpt_provider import DocsGPTAPILLM
 from application.llm.premai import PremAILLM
+from application.llm.google_ai import GoogleLLM
 
 
 class LLMCreator:
@@ -17,6 +19,8 @@ class LLMCreator:
         "anthropic": AnthropicLLM,
         "docsgpt": DocsGPTAPILLM,
         "premai": PremAILLM,
+        "groq": GroqLLM,
+        "google": GoogleLLM
     }
 
     @classmethod

application/llm/openai.py

@@ -1,40 +1,40 @@
-from application.llm.base import BaseLLM
 from application.core.settings import settings
+from application.llm.base import BaseLLM
 
 
 class OpenAILLM(BaseLLM):
 
     def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
-        global openai
         from openai import OpenAI
 
         super().__init__(*args, **kwargs)
-        self.client = OpenAI(
-            api_key=api_key,
-        )
+        if settings.OPENAI_BASE_URL:
+            self.client = OpenAI(api_key=api_key, base_url=settings.OPENAI_BASE_URL)
+        else:
+            self.client = OpenAI(api_key=api_key)
         self.api_key = api_key
         self.user_api_key = user_api_key
 
-    def _get_openai(self):
-        # Import openai when needed
-        import openai
-
-        return openai
-
     def _raw_gen(
         self,
         baseself,
         model,
         messages,
         stream=False,
+        tools=None,
         engine=settings.AZURE_DEPLOYMENT_NAME,
-        **kwargs
+        **kwargs,
     ):
-        response = self.client.chat.completions.create(
-            model=model, messages=messages, stream=stream, **kwargs
-        )
-
-        return response.choices[0].message.content
+        if tools:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, tools=tools, **kwargs
+            )
+            return response.choices[0]
+        else:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, **kwargs
+            )
+            return response.choices[0].message.content
 
     def _raw_gen_stream(
         self,
@@ -42,19 +42,21 @@ class OpenAILLM(BaseLLM):
         model,
         messages,
         stream=True,
+        tools=None,
         engine=settings.AZURE_DEPLOYMENT_NAME,
-        **kwargs
+        **kwargs,
     ):
         response = self.client.chat.completions.create(
             model=model, messages=messages, stream=stream, **kwargs
         )
 
         for line in response:
-            # import sys
-            # print(line.choices[0].delta.content, file=sys.stderr)
             if line.choices[0].delta.content is not None:
                 yield line.choices[0].delta.content
 
+    def _supports_tools(self):
+        return True
+
 
 class AzureOpenAILLM(OpenAILLM):
 
@@ -73,8 +75,3 @@ class AzureOpenAILLM(OpenAILLM):
             api_base=settings.OPENAI_API_BASE,
             deployment_name=settings.AZURE_DEPLOYMENT_NAME,
         )
-
-    def _get_openai(self):
-        openai = super()._get_openai()
-
-        return openai

application/llm/sagemaker.py

@@ -76,7 +76,7 @@ class SagemakerAPILLM(BaseLLM):
         self.endpoint = settings.SAGEMAKER_ENDPOINT
         self.runtime = runtime
 
-    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+    def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs):
         context = messages[0]["content"]
         user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
@@ -105,7 +105,7 @@ class SagemakerAPILLM(BaseLLM):
         print(result[0]["generated_text"], file=sys.stderr)
         return result[0]["generated_text"][len(prompt) :]
 
-    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, tools=None, **kwargs):
         context = messages[0]["content"]
         user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"

application/parser/chunking.py

@@ -0,0 +1,118 @@
+import re
+from typing import List, Tuple
+import logging
+from application.parser.schema.base import Document
+from application.utils import get_encoding
+
+logger = logging.getLogger(__name__)
+
+class Chunker:
+    def __init__(
+        self,
+        chunking_strategy: str = "classic_chunk",
+        max_tokens: int = 2000,
+        min_tokens: int = 150,
+        duplicate_headers: bool = False,
+    ):
+        if chunking_strategy not in ["classic_chunk"]:
+            raise ValueError(f"Unsupported chunking strategy: {chunking_strategy}")
+        self.chunking_strategy = chunking_strategy
+        self.max_tokens = max_tokens
+        self.min_tokens = min_tokens
+        self.duplicate_headers = duplicate_headers
+        self.encoding = get_encoding()
+
+    def separate_header_and_body(self, text: str) -> Tuple[str, str]:
+        header_pattern = r"^(.*?\n){3}"
+        match = re.match(header_pattern, text)
+        if match:
+            header = match.group(0)
+            body = text[len(header):]
+        else:
+            header, body = "", text  # No header, treat entire text as body
+        return header, body
+
+    def combine_documents(self, doc: Document, next_doc: Document) -> Document:
+        combined_text = doc.text + " " + next_doc.text
+        combined_token_count = len(self.encoding.encode(combined_text))
+        new_doc = Document(
+            text=combined_text,
+            doc_id=doc.doc_id,
+            embedding=doc.embedding,
+            extra_info={**(doc.extra_info or {}), "token_count": combined_token_count}
+        )
+        return new_doc
+    
+    def split_document(self, doc: Document) -> List[Document]:
+        split_docs = []
+        header, body = self.separate_header_and_body(doc.text)
+        header_tokens = self.encoding.encode(header) if header else []
+        body_tokens = self.encoding.encode(body)
+
+        current_position = 0
+        part_index = 0
+        while current_position < len(body_tokens):
+            end_position = current_position + self.max_tokens - len(header_tokens)
+            chunk_tokens = (header_tokens + body_tokens[current_position:end_position]
+                            if self.duplicate_headers or part_index == 0 else body_tokens[current_position:end_position])
+            chunk_text = self.encoding.decode(chunk_tokens)
+            new_doc = Document(
+                text=chunk_text,
+                doc_id=f"{doc.doc_id}-{part_index}",
+                embedding=doc.embedding,
+                extra_info={**(doc.extra_info or {}), "token_count": len(chunk_tokens)}
+            )
+            split_docs.append(new_doc)
+            current_position = end_position
+            part_index += 1
+            header_tokens = []
+        return split_docs
+
+    def classic_chunk(self, documents: List[Document]) -> List[Document]:
+        processed_docs = []
+        i = 0
+        while i < len(documents):
+            doc = documents[i]
+            tokens = self.encoding.encode(doc.text)
+            token_count = len(tokens)
+
+            if self.min_tokens <= token_count <= self.max_tokens:
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            elif token_count < self.min_tokens:
+                if i + 1 < len(documents):
+                    next_doc = documents[i + 1]
+                    next_tokens = self.encoding.encode(next_doc.text)
+                    if token_count + len(next_tokens) <= self.max_tokens:
+                        # Combine small documents
+                        combined_doc = self.combine_documents(doc, next_doc)
+                        processed_docs.append(combined_doc)
+                        i += 2
+                    else:
+                        # Keep the small document as is if adding next_doc would exceed max_tokens
+                        doc.extra_info = doc.extra_info or {}
+                        doc.extra_info["token_count"] = token_count
+                        processed_docs.append(doc)
+                        i += 1
+                else:
+                    # No next document to combine with; add the small document as is
+                    doc.extra_info = doc.extra_info or {}
+                    doc.extra_info["token_count"] = token_count
+                    processed_docs.append(doc)
+                    i += 1
+            else:
+                # Split large documents
+                processed_docs.extend(self.split_document(doc))
+                i += 1
+        return processed_docs
+
+    def chunk(
+        self,
+        documents: List[Document]
+    ) -> List[Document]:
+        if self.chunking_strategy == "classic_chunk":
+            return self.classic_chunk(documents)
+        else:
+            raise ValueError("Unsupported chunking strategy")

application/parser/embedding_pipeline.py

@@ -0,0 +1,86 @@
+import os
+import logging
+from retry import retry
+from tqdm import tqdm
+from application.core.settings import settings
+from application.vectorstore.vector_creator import VectorCreator
+
+
+@retry(tries=10, delay=60)
+def add_text_to_store_with_retry(store, doc, source_id):
+    """
+    Add a document's text and metadata to the vector store with retry logic.
+    Args:
+        store: The vector store object.
+        doc: The document to be added.
+        source_id: Unique identifier for the source.
+    """
+    try:
+        doc.metadata["source_id"] = str(source_id)
+        store.add_texts([doc.page_content], metadatas=[doc.metadata])
+    except Exception as e:
+        logging.error(f"Failed to add document with retry: {e}")
+        raise
+
+
+def embed_and_store_documents(docs, folder_name, source_id, task_status):
+    """
+    Embeds documents and stores them in a vector store.
+
+    Args:
+        docs (list): List of documents to be embedded and stored.
+        folder_name (str): Directory to save the vector store.
+        source_id (str): Unique identifier for the source.
+        task_status: Task state manager for progress updates.
+
+    Returns:
+        None
+    """
+    # Ensure the folder exists
+    if not os.path.exists(folder_name):
+        os.makedirs(folder_name)
+
+    # Initialize vector store
+    if settings.VECTOR_STORE == "faiss":
+        docs_init = [docs.pop(0)]
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            docs_init=docs_init,
+            source_id=folder_name,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+    else:
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            source_id=source_id,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+        store.delete_index()
+
+    total_docs = len(docs)
+
+    # Process and embed documents
+    for idx, doc in tqdm(
+        enumerate(docs),
+        desc="Embedding 🦖",
+        unit="docs",
+        total=total_docs,
+        bar_format="{l_bar}{bar}| Time Left: {remaining}",
+    ):
+        try:
+            # Update task status for progress tracking
+            progress = int(((idx + 1) / total_docs) * 100)
+            task_status.update_state(state="PROGRESS", meta={"current": progress})
+
+            # Add document to vector store
+            add_text_to_store_with_retry(store, doc, source_id)
+        except Exception as e:
+            logging.error(f"Error embedding document {idx}: {e}")
+            logging.info(f"Saving progress at document {idx} out of {total_docs}")
+            store.save_local(folder_name)
+            break
+
+    # Save the vector store
+    if settings.VECTOR_STORE == "faiss":
+        store.save_local(folder_name)
+    logging.info("Vector store saved successfully.")

application/parser/file/bulk.py

@@ -10,18 +10,27 @@ from application.parser.file.epub_parser import EpubParser
 from application.parser.file.html_parser import HTMLParser
 from application.parser.file.markdown_parser import MarkdownParser
 from application.parser.file.rst_parser import RstParser
-from application.parser.file.tabular_parser import PandasCSVParser
+from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser
+from application.parser.file.json_parser import JSONParser
+from application.parser.file.pptx_parser import PPTXParser
+from application.parser.file.image_parser import ImageParser
 from application.parser.schema.base import Document
 
 DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
     ".pdf": PDFParser(),
     ".docx": DocxParser(),
     ".csv": PandasCSVParser(),
+    ".xlsx":ExcelParser(),
     ".epub": EpubParser(),
     ".md": MarkdownParser(),
     ".rst": RstParser(),
     ".html": HTMLParser(),
     ".mdx": MarkdownParser(),
+    ".json":JSONParser(),
+    ".pptx":PPTXParser(),
+    ".png": ImageParser(),
+    ".jpg": ImageParser(),
+    ".jpeg": ImageParser(),
 }
 
 

application/parser/file/docs_parser.py

@@ -7,7 +7,8 @@ from pathlib import Path
 from typing import Dict
 
 from application.parser.file.base_parser import BaseParser
-
+from application.core.settings import settings
+import requests
 
 class PDFParser(BaseParser):
     """PDF parser."""
@@ -18,6 +19,15 @@ class PDFParser(BaseParser):
 
     def parse_file(self, file: Path, errors: str = "ignore") -> str:
         """Parse file."""
+        if settings.PARSE_PDF_AS_IMAGE:
+            doc2md_service = "https://llm.arc53.com/doc2md"
+            # alternatively you can use local vision capable LLM
+            with open(file, "rb") as file_loaded:
+                files = {'file': file_loaded}
+                response = requests.post(doc2md_service, files=files)   
+                data = response.json()["markdown"] 
+            return data
+
         try:
             import PyPDF2
         except ImportError:

application/parser/file/html_parser.py

@@ -3,7 +3,6 @@
 Contains parser for html files.
 
 """
-import re
 from pathlib import Path
 from typing import Dict, Union
 
@@ -18,66 +17,8 @@ class HTMLParser(BaseParser):
         return {}
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
-        """Parse file.
+        from langchain_community.document_loaders import BSHTMLLoader
 
-            Returns:
-            Union[str, List[str]]: a string or a List of strings.
-        """
-        try:
-            from unstructured.partition.html import partition_html
-            from unstructured.staging.base import convert_to_isd
-            from unstructured.cleaners.core import clean
-        except ImportError:
-            raise ValueError("unstructured package is required to parse HTML files.")
-
-        # Using the unstructured library to convert the html to isd format
-        # isd sample : isd = [
-        #   {"text": "My Title", "type": "Title"},
-        #   {"text": "My Narrative", "type": "NarrativeText"}
-        # ]
-        with open(file, "r", encoding="utf-8") as fp:
-            elements = partition_html(file=fp)
-            isd = convert_to_isd(elements)
-
-            # Removing non ascii charactwers from isd_el['text']
-        for isd_el in isd:
-            isd_el['text'] = isd_el['text'].encode("ascii", "ignore").decode()
-
-        # Removing all the \n characters from isd_el['text'] using regex and replace with single space
-        # Removing all the extra spaces  from isd_el['text'] using regex and replace with single space
-        for isd_el in isd:
-            isd_el['text'] = re.sub(r'\n', ' ', isd_el['text'], flags=re.MULTILINE | re.DOTALL)
-            isd_el['text'] = re.sub(r"\s{2,}", " ", isd_el['text'], flags=re.MULTILINE | re.DOTALL)
-
-        # more cleaning: extra_whitespaces, dashes, bullets, trailing_punctuation
-        for isd_el in isd:
-            clean(isd_el['text'], extra_whitespace=True, dashes=True, bullets=True, trailing_punctuation=True)
-
-        # Creating a list of all the indexes of isd_el['type'] = 'Title'
-        title_indexes = [i for i, isd_el in enumerate(isd) if isd_el['type'] == 'Title']
-
-        # Creating 'Chunks' - List of lists of strings 
-        # each list starting with isd_el['type'] = 'Title' and all the data till the next 'Title'
-        # Each Chunk can be thought of as an individual set of data, which can be sent to the model
-        # Where Each Title is grouped together with the data under it
-
-        Chunks = [[]]
-        final_chunks = list(list())
-
-        for i, isd_el in enumerate(isd):
-            if i in title_indexes:
-                Chunks.append([])
-            Chunks[-1].append(isd_el['text'])
-
-        # Removing all the chunks with sum of length of all the strings in the chunk < 25
-        # TODO: This value can be an user defined variable
-        for chunk in Chunks:
-            # sum of length of all the strings in the chunk
-            sum = 0
-            sum += len(str(chunk))
-            if sum < 25:
-                Chunks.remove(chunk)
-            else:
-                # appending all the approved chunks to final_chunks as a single string       
-                final_chunks.append(" ".join([str(item) for item in chunk]))
-        return final_chunks
+        loader = BSHTMLLoader(file)
+        data = loader.load()        
+        return data

application/parser/file/image_parser.py

@@ -0,0 +1,27 @@
+"""Image parser.
+
+Contains parser for .png, .jpg, .jpeg files.
+
+"""
+from pathlib import Path
+import requests
+from typing import Dict, Union
+
+from application.parser.file.base_parser import BaseParser
+
+
+class ImageParser(BaseParser):
+    """Image parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        doc2md_service = "https://llm.arc53.com/doc2md"
+        # alternatively you can use local vision capable LLM
+        with open(file, "rb") as file_loaded:
+            files = {'file': file_loaded}
+            response = requests.post(doc2md_service, files=files)   
+            data = response.json()["markdown"] 
+        return data

application/parser/file/json_parser.py

@@ -0,0 +1,57 @@
+import json
+from typing import Any, Dict, List, Union
+from pathlib import Path
+
+from application.parser.file.base_parser import BaseParser
+
+class JSONParser(BaseParser):
+    r"""JSON (.json) parser.
+
+    Parses JSON files into a list of strings or a concatenated document.
+    It handles both JSON objects (dictionaries) and arrays (lists).
+
+    Args:
+        concat_rows (bool): Whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each item in the JSON.
+            True by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        json_config (dict): Options for parsing JSON. Can be used to specify options like
+        custom decoding or formatting. Set to empty dict by default.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            row_joiner: str = "\n",
+            json_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._row_joiner = row_joiner
+        self._json_config = json_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse JSON file."""
+        
+        with open(file, 'r', encoding='utf-8') as f:
+                data = json.load(f, **self._json_config)
+
+        if isinstance(data, dict):
+            data = [data]
+
+        if self._concat_rows:
+            return self._row_joiner.join([str(item) for item in data])
+        else:
+            return data

application/parser/file/pptx_parser.py

@@ -0,0 +1,75 @@
+"""PPT parser.
+Contains parsers for presentation (.pptx) files to extract slide text.
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from application.parser.file.base_parser import BaseParser
+
+class PPTXParser(BaseParser):
+    r"""PPTX (.pptx) parser for extracting text from PowerPoint slides.
+    Args:
+        concat_slides (bool): Specifies whether to concatenate all slide text into one document.
+            - If True, slide texts will be joined together as a single string.
+            - If False, each slide's text will be stored as a separate entry in a list.
+            Set to True by default.
+        slide_separator (str): Separator used to join slides' text content.
+            Only used when `concat_slides=True`. Default is "\n".
+        Refer to https://python-pptx.readthedocs.io/en/latest/ for more information.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        concat_slides: bool = True,
+        slide_separator: str = "\n",
+        **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_slides = concat_slides
+        self._slide_separator = slide_separator
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        r"""
+        Parse a .pptx file and extract text from each slide.
+        Args:
+            file (Path): Path to the .pptx file.
+            errors (str): Error handling policy ('ignore' by default).
+        Returns:
+            Union[str, List[str]]: Concatenated text if concat_slides is True,
+            otherwise a list of slide texts.
+        """
+
+        try:
+            from pptx import Presentation
+        except ImportError:
+            raise ImportError("pptx module is required to read .PPTX files.")
+
+        try:
+            presentation = Presentation(file)
+            slide_texts=[]
+
+            # Iterate over each slide in the presentation
+            for slide in presentation.slides:
+                slide_text=""
+
+                # Iterate over each shape in the slide
+                for shape in slide.shapes:
+                    # Check if the shape has a 'text' attribute and append that to the slide_text
+                    if hasattr(shape,"text"):
+                        slide_text+=shape.text
+
+                slide_texts.append(slide_text.strip())
+
+            if self._concat_slides:
+                return self._slide_separator.join(slide_texts)
+            else:
+                return slide_texts
+
+        except Exception as e:
+            raise e

application/parser/file/rst_parser.py

@@ -91,6 +91,25 @@ class RstParser(BaseParser):
             ]
         return rst_tups
 
+    def chunk_by_token_count(self, text: str, max_tokens: int = 100) -> List[str]:
+        """Chunk text by token count."""
+    
+        avg_token_length = 5
+    
+        chunk_size = max_tokens * avg_token_length
+
+        chunks = []
+        for i in range(0, len(text), chunk_size):
+            chunk = text[i:i+chunk_size]
+            if i + chunk_size < len(text):
+                last_space = chunk.rfind(' ')
+                if last_space != -1:
+                    chunk = chunk[:last_space]
+            
+            chunks.append(chunk.strip())
+        
+        return chunks
+    
     def remove_images(self, content: str) -> str:
         pattern = r"\.\. image:: (.*)"
         content = re.sub(pattern, "", content)
@@ -136,7 +155,7 @@ class RstParser(BaseParser):
         return {}
 
     def parse_tups(
-            self, filepath: Path, errors: str = "ignore"
+            self, filepath: Path, errors: str = "ignore",max_tokens: Optional[int] = 1000
     ) -> List[Tuple[Optional[str], str]]:
         """Parse file into tuples."""
         with open(filepath, "r") as f:
@@ -156,6 +175,15 @@ class RstParser(BaseParser):
             rst_tups = self.remove_whitespaces_excess(rst_tups)
         if self._remove_characters_excess:
             rst_tups = self.remove_characters_excess(rst_tups)
+
+        # Apply chunking if max_tokens is provided
+        if max_tokens is not None:
+            chunked_tups = []
+            for header, text in rst_tups:
+                chunks = self.chunk_by_token_count(text, max_tokens)
+                for idx, chunk in enumerate(chunks):
+                    chunked_tups.append((f"{header} - Chunk {idx + 1}", chunk))
+            return chunked_tups    
         return rst_tups
 
     def parse_file(

application/parser/file/tabular_parser.py

@@ -113,3 +113,68 @@ class PandasCSVParser(BaseParser):
             return (self._row_joiner).join(text_list)
         else:
             return text_list
+
+
+class ExcelParser(BaseParser):
+    r"""Excel (.xlsx) parser.
+
+    Parses Excel files using Pandas `read_excel` function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_excel` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the table structure on its own.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read Excel files.")
+
+        df = pd.read_excel(file, **self._pandas_config)
+
+        text_list = df.apply(
+            lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
+        ).tolist()
+
+        if self._concat_rows:
+            return (self._row_joiner).join(text_list)
+        else:
+            return text_list

application/parser/java2doc.py

@@ -1,66 +0,0 @@
-import os
-
-import javalang
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.java'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, "r") as file:
-        java_code = file.read()
-        methods = {}
-        tree = javalang.parse.parse(java_code)
-        for _, node in tree.filter(javalang.tree.MethodDeclaration):
-            method_name = node.name
-            start_line = node.position.line - 1
-            end_line = start_line
-            brace_count = 0
-            for line in java_code.splitlines()[start_line:]:
-                end_line += 1
-                brace_count += line.count("{") - line.count("}")
-                if brace_count == 0:
-                    break
-            method_source_code = "\n".join(java_code.splitlines()[start_line:end_line])
-            methods[method_name] = method_source_code
-    return methods
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = javalang.parse.parse(source_code)
-        for class_decl in tree.types:
-            class_name = class_decl.name
-            declarations = []
-            methods = []
-            for field_decl in class_decl.fields:
-                field_name = field_decl.declarators[0].name
-                field_type = field_decl.type.name
-                declarations.append(f"{field_type} {field_name}")
-            for method_decl in class_decl.methods:
-                methods.append(method_decl.name)
-            class_string = "Declarations: " + ", ".join(declarations) + "\n  Method name: " + ", ".join(methods)
-            classes[class_name] = class_string
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict

application/parser/js2doc.py

@@ -1,70 +0,0 @@
-import os
-
-import escodegen
-import esprima
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.js'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        functions = {}
-        tree = esprima.parseScript(source_code)
-        for node in tree.body:
-            if node.type == 'FunctionDeclaration':
-                func_name = node.id.name if node.id else '<anonymous>'
-                functions[func_name] = escodegen.generate(node)
-            elif node.type == 'VariableDeclaration':
-                for declaration in node.declarations:
-                    if declaration.init and declaration.init.type == 'FunctionExpression':
-                        func_name = declaration.id.name if declaration.id else '<anonymous>'
-                        functions[func_name] = escodegen.generate(declaration.init)
-            elif node.type == 'ClassDeclaration':
-                for subnode in node.body.body:
-                    if subnode.type == 'MethodDefinition':
-                        func_name = subnode.key.name
-                        functions[func_name] = escodegen.generate(subnode.value)
-                    elif subnode.type == 'VariableDeclaration':
-                        for declaration in subnode.declarations:
-                            if declaration.init and declaration.init.type == 'FunctionExpression':
-                                func_name = declaration.id.name if declaration.id else '<anonymous>'
-                                functions[func_name] = escodegen.generate(declaration.init)
-        return functions
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = esprima.parseScript(source_code)
-        for node in tree.body:
-            if node.type == 'ClassDeclaration':
-                class_name = node.id.name
-                function_names = []
-                for subnode in node.body.body:
-                    if subnode.type == 'MethodDefinition':
-                        function_names.append(subnode.key.name)
-                classes[class_name] = ", ".join(function_names)
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict

application/parser/open_ai_func.py

@@ -1,72 +0,0 @@
-import os
-
-from application.vectorstore.vector_creator import VectorCreator
-from application.core.settings import settings
-from retry import retry
-
-
-# from langchain_community.embeddings import HuggingFaceEmbeddings
-# from langchain_community.embeddings import HuggingFaceInstructEmbeddings
-# from langchain_community.embeddings import CohereEmbeddings
-
-
-@retry(tries=10, delay=60)
-def store_add_texts_with_retry(store, i):
-    store.add_texts([i.page_content], metadatas=[i.metadata])
-    # store_pine.add_texts([i.page_content], metadatas=[i.metadata])
-
-
-def call_openai_api(docs, folder_name, task_status):
-    # Function to create a vector store from the documents and save it to disk
-
-    if not os.path.exists(f"{folder_name}"):
-        os.makedirs(f"{folder_name}")
-
-    from tqdm import tqdm
-
-    c1 = 0
-    if settings.VECTOR_STORE == "faiss":
-        docs_init = [docs[0]]
-        docs.pop(0)
-
-        store = VectorCreator.create_vectorstore(
-            settings.VECTOR_STORE,
-            docs_init=docs_init,
-            path=f"{folder_name}",
-            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
-        )
-    else:
-        store = VectorCreator.create_vectorstore(
-            settings.VECTOR_STORE,
-            path=f"{folder_name}",
-            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
-        )
-    # Uncomment for MPNet embeddings
-    # model_name = "sentence-transformers/all-mpnet-base-v2"
-    # hf = HuggingFaceEmbeddings(model_name=model_name)
-    # store = FAISS.from_documents(docs_test, hf)
-    s1 = len(docs)
-    for i in tqdm(
-        docs,
-        desc="Embedding 🦖",
-        unit="docs",
-        total=len(docs),
-        bar_format="{l_bar}{bar}| Time Left: {remaining}",
-    ):
-        try:
-            task_status.update_state(
-                state="PROGRESS", meta={"current": int((c1 / s1) * 100)}
-            )
-            store_add_texts_with_retry(store, i)
-        except Exception as e:
-            print(e)
-            print("Error on ", i)
-            print("Saving progress")
-            print(f"stopped at {c1} out of {len(docs)}")
-            store.save_local(f"{folder_name}")
-            break
-        c1 += 1
-    if settings.VECTOR_STORE == "faiss":
-        store.save_local(f"{folder_name}")
-
-

application/parser/py2doc.py

@@ -1,121 +0,0 @@
-import ast
-import os
-from pathlib import Path
-
-import tiktoken
-from langchain.llms import OpenAI
-from langchain.prompts import PromptTemplate
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.py'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        functions = {}
-        tree = ast.parse(source_code)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.FunctionDef):
-                func_name = node.name
-                func_def = ast.get_source_segment(source_code, node)
-                functions[func_name] = func_def
-    return functions
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = ast.parse(source_code)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.ClassDef):
-                class_name = node.name
-                function_names = []
-                for subnode in ast.walk(node):
-                    if isinstance(subnode, ast.FunctionDef):
-                        function_names.append(subnode.name)
-                classes[class_name] = ", ".join(function_names)
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict
-
-
-def parse_functions(functions_dict, formats, dir):
-    c1 = len(functions_dict)
-    for i, (source, functions) in enumerate(functions_dict.items(), start=1):
-        print(f"Processing file {i}/{c1}")
-        source_w = source.replace(dir + "/", "").replace("." + formats, ".md")
-        subfolders = "/".join(source_w.split("/")[:-1])
-        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
-        for j, (name, function) in enumerate(functions.items(), start=1):
-            print(f"Processing function {j}/{len(functions)}")
-            prompt = PromptTemplate(
-                input_variables=["code"],
-                template="Code: \n{code}, \nDocumentation: ",
-            )
-            llm = OpenAI(temperature=0)
-            response = llm(prompt.format(code=function))
-            mode = "a" if Path(f"outputs/{source_w}").exists() else "w"
-            with open(f"outputs/{source_w}", mode) as f:
-                f.write(
-                    f"\n\n# Function name: {name} \n\nFunction: \n```\n{function}\n```, \nDocumentation: \n{response}")
-
-
-def parse_classes(classes_dict, formats, dir):
-    c1 = len(classes_dict)
-    for i, (source, classes) in enumerate(classes_dict.items()):
-        print(f"Processing file {i + 1}/{c1}")
-        source_w = source.replace(dir + "/", "").replace("." + formats, ".md")
-        subfolders = "/".join(source_w.split("/")[:-1])
-        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
-        for name, function_names in classes.items():
-            print(f"Processing Class {i + 1}/{c1}")
-            prompt = PromptTemplate(
-                input_variables=["class_name", "functions_names"],
-                template="Class name: {class_name} \nFunctions: {functions_names}, \nDocumentation: ",
-            )
-            llm = OpenAI(temperature=0)
-            response = llm(prompt.format(class_name=name, functions_names=function_names))
-
-            with open(f"outputs/{source_w}", "a" if Path(f"outputs/{source_w}").exists() else "w") as f:
-                f.write(f"\n\n# Class name: {name} \n\nFunctions: \n{function_names}, \nDocumentation: \n{response}")
-
-
-def transform_to_docs(functions_dict, classes_dict, formats, dir):
-    docs_content = ''.join([str(key) + str(value) for key, value in functions_dict.items()])
-    docs_content += ''.join([str(key) + str(value) for key, value in classes_dict.items()])
-
-    num_tokens = len(tiktoken.get_encoding("cl100k_base").encode(docs_content))
-    total_price = ((num_tokens / 1000) * 0.02)
-
-    print(f"Number of Tokens = {num_tokens:,d}")
-    print(f"Approx Cost = ${total_price:,.2f}")
-
-    user_input = input("Price Okay? (Y/N)\n").lower()
-    if user_input == "y" or user_input == "":
-        if not Path("outputs").exists():
-            Path("outputs").mkdir()
-        parse_functions(functions_dict, formats, dir)
-        parse_classes(classes_dict, formats, dir)
-        print("All done!")
-    else:
-        print("The API was not called. No money was spent.")

application/parser/remote/crawler_loader.py

@@ -2,16 +2,16 @@ import requests
 from urllib.parse import urlparse, urljoin
 from bs4 import BeautifulSoup
 from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
+from langchain_community.document_loaders import WebBaseLoader
 
 class CrawlerLoader(BaseRemote):
     def __init__(self, limit=10):
-        from langchain.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader  # Initialize the document loader
         self.limit = limit  # Set the limit for the number of pages to scrape
 
     def load_data(self, inputs):
         url = inputs
-        # Check if the input is a list and if it is, use the first element
         if isinstance(url, list) and url:
             url = url[0]
 
@@ -19,24 +19,29 @@ class CrawlerLoader(BaseRemote):
         if not urlparse(url).scheme:
             url = "http://" + url
 
-        visited_urls = set()  # Keep track of URLs that have been visited
-        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname  # Extract the base URL
-        urls_to_visit = [url]  # List of URLs to be visited, starting with the initial URL
-        loaded_content = []  # Store the loaded content from each URL
+        visited_urls = set()
+        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname
+        urls_to_visit = [url]
+        loaded_content = []
 
-        # Continue crawling until there are no more URLs to visit
         while urls_to_visit:
-            current_url = urls_to_visit.pop(0)  # Get the next URL to visit
-            visited_urls.add(current_url)  # Mark the URL as visited
+            current_url = urls_to_visit.pop(0)
+            visited_urls.add(current_url)
 
-            # Try to load and process the content from the current URL
             try:
-                response = requests.get(current_url)  # Fetch the content of the current URL
-                response.raise_for_status()  # Raise an exception for HTTP errors
-                loader = self.loader([current_url])  # Initialize the document loader for the current URL
-                loaded_content.extend(loader.load())  # Load the content and add it to the loaded_content list
+                response = requests.get(current_url)
+                response.raise_for_status()
+                loader = self.loader([current_url])
+                docs = loader.load()
+                # Convert the loaded documents to your Document schema
+                for doc in docs:
+                    loaded_content.append(
+                        Document(
+                            doc.page_content,
+                            extra_info=doc.metadata
+                        )
+                    )
             except Exception as e:
-                # Print an error message if loading or processing fails and continue with the next URL
                 print(f"Error processing URL {current_url}: {e}")
                 continue
 
@@ -45,15 +50,15 @@ class CrawlerLoader(BaseRemote):
             all_links = [
                 urljoin(current_url, a['href'])
                 for a in soup.find_all('a', href=True)
-                if base_url in urljoin(current_url, a['href'])  # Ensure links are from the same domain
+                if base_url in urljoin(current_url, a['href'])
             ]
 
             # Add new links to the list of URLs to visit if they haven't been visited yet
             urls_to_visit.extend([link for link in all_links if link not in visited_urls])
-            urls_to_visit = list(set(urls_to_visit))  # Remove duplicate URLs
+            urls_to_visit = list(set(urls_to_visit))
 
             # Stop crawling if the limit of pages to scrape is reached
             if self.limit is not None and len(visited_urls) >= self.limit:
                 break
 
-        return loaded_content  # Return the loaded content from all visited URLs
+        return loaded_content

application/parser/remote/crawler_markdown.py

@@ -0,0 +1,139 @@
+import requests
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup
+from application.parser.remote.base import BaseRemote
+import re
+from markdownify import markdownify
+from application.parser.schema.base import Document
+import tldextract
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10, allow_subdomains=False):
+        """
+        Given a URL crawl web pages up to `self.limit`,
+        convert HTML content to Markdown, and returning a list of Document objects.
+
+        :param limit: The maximum number of pages to crawl.
+        :param allow_subdomains: If True, crawl pages on subdomains of the base domain.
+        """
+        self.limit = limit
+        self.allow_subdomains = allow_subdomains
+        self.session = requests.Session()
+
+    def load_data(self, inputs):
+        url = inputs
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Ensure the URL has a scheme (if not, default to http)
+        if not urlparse(url).scheme:
+            url = "http://" + url
+
+        # Keep track of visited URLs to avoid revisiting the same page
+        visited_urls = set()
+
+        # Determine the base domain for link filtering using tldextract
+        base_domain = self._get_base_domain(url)
+        urls_to_visit = {url}
+        documents = []
+
+        while urls_to_visit:
+            current_url = urls_to_visit.pop()
+
+            # Skip if already visited
+            if current_url in visited_urls:
+                continue
+            visited_urls.add(current_url)
+
+            # Fetch the page content
+            html_content = self._fetch_page(current_url)
+            if html_content is None:
+                continue
+
+            # Convert the HTML to Markdown for cleaner text formatting
+            title, language, processed_markdown = self._process_html_to_markdown(html_content, current_url)
+            if processed_markdown:
+                # Create a Document for each visited page
+                documents.append(
+                    Document(
+                        processed_markdown,  # content
+                        None,  # doc_id
+                        None,  # embedding
+                        {"source": current_url, "title": title, "language": language} # extra_info
+                    )
+                )
+
+            # Extract links and filter them according to domain rules
+            new_links = self._extract_links(html_content, current_url)
+            filtered_links = self._filter_links(new_links, base_domain)
+
+            # Add any new, not-yet-visited links to the queue
+            urls_to_visit.update(link for link in filtered_links if link not in visited_urls)
+
+            # If we've reached the limit, stop crawling
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return documents
+
+    def _fetch_page(self, url):
+        try:
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+            return response.text
+        except requests.exceptions.RequestException as e:
+            print(f"Error fetching URL {url}: {e}")
+            return None
+
+    def _process_html_to_markdown(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        title_tag = soup.find('title')
+        title = title_tag.text.strip() if title_tag else "No Title"
+
+        # Extract language
+        language_tag = soup.find('html')
+        language = language_tag.get('lang', 'en') if language_tag else "en"
+
+        markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
+        # Reduce sequences of more than two newlines to exactly three
+        markdownified = re.sub(r'\n{3,}', '\n\n\n', markdownified)
+        return title, language, markdownified
+
+    def _extract_links(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        links = []
+        for a in soup.find_all('a', href=True):
+            full_url = urljoin(current_url, a['href'])
+            links.append((full_url, a.text.strip()))
+        return links
+
+    def _get_base_domain(self, url):
+        extracted = tldextract.extract(url)
+        # Reconstruct the domain as domain.suffix
+        base_domain = f"{extracted.domain}.{extracted.suffix}"
+        return base_domain
+
+    def _filter_links(self, links, base_domain):
+        """
+        Filter the extracted links to only include those that match the crawling criteria:
+        - If allow_subdomains is True, allow any link whose domain ends with the base_domain.
+        - If allow_subdomains is False, only allow exact matches of the base_domain.
+        """
+        filtered = []
+        for link, _ in links:
+            parsed_link = urlparse(link)
+            if not parsed_link.netloc:
+                continue
+
+            extracted = tldextract.extract(parsed_link.netloc)
+            link_base = f"{extracted.domain}.{extracted.suffix}"
+
+            if self.allow_subdomains:
+                # For subdomains: sub.example.com ends with example.com
+                if link_base == base_domain or link_base.endswith("." + base_domain):
+                    filtered.append(link)
+            else:
+                # Exact domain match
+                if link_base == base_domain:
+                    filtered.append(link)
+        return filtered

application/parser/remote/github_loader.py

@@ -0,0 +1,58 @@
+import base64
+import requests
+from typing import List
+from application.parser.remote.base import BaseRemote
+from langchain_core.documents import Document
+import mimetypes
+
+class GitHubLoader(BaseRemote):
+    def __init__(self):
+        self.access_token = None
+        self.headers = {
+            "Authorization": f"token {self.access_token}"
+        } if self.access_token else {}
+        return
+
+    def fetch_file_content(self, repo_url: str, file_path: str) -> str:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
+        response = requests.get(url, headers=self.headers)
+
+        if response.status_code == 200:
+            content = response.json()
+            mime_type, _ = mimetypes.guess_type(file_path)  # Guess the MIME type based on the file extension
+
+            if content.get("encoding") == "base64":
+                if mime_type and mime_type.startswith("text"):  # Handle only text files
+                    try:
+                        decoded_content = base64.b64decode(content["content"]).decode("utf-8")
+                        return f"Filename: {file_path}\n\n{decoded_content}"
+                    except Exception as e:
+                        raise e
+                else:
+                    return f"Filename: {file_path} is a binary file and was skipped."
+            else:
+                return f"Filename: {file_path}\n\n{content['content']}"
+        else:
+            response.raise_for_status()
+
+    def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
+        response = requests.get(url, headers={**self.headers, "Accept": "application/vnd.github.v3.raw"})
+        contents = response.json()
+        files = []
+        for item in contents:
+            if item["type"] == "file":
+                files.append(item["path"])
+            elif item["type"] == "dir":
+                files.extend(self.fetch_repo_files(repo_url, item["path"]))
+        return files
+
+    def load_data(self, repo_url: str) -> List[Document]:
+        repo_name = repo_url.split("github.com/")[-1]
+        files = self.fetch_repo_files(repo_name)
+        documents = []
+        for file_path in files:
+            content = self.fetch_file_content(repo_name, file_path)
+            documents.append(Document(page_content=content, metadata={"title": file_path, 
+            "source": f"https://github.com/{repo_name}/blob/main/{file_path}"}))
+        return documents

application/parser/remote/reddit_loader.py

@@ -1,10 +1,19 @@
 from application.parser.remote.base import BaseRemote
 from langchain_community.document_loaders import RedditPostsLoader
+import json
 
 
 class RedditPostsLoaderRemote(BaseRemote):
     def load_data(self, inputs):
-        data = eval(inputs)
+        try:
+            data = json.loads(inputs)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON input: {e}")
+
+        required_fields = ["client_id", "client_secret", "user_agent", "search_queries"]
+        missing_fields = [field for field in required_fields if field not in data]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
         client_id = data.get("client_id")
         client_secret = data.get("client_secret")
         user_agent = data.get("user_agent")

application/parser/remote/remote_creator.py

@@ -2,6 +2,7 @@ from application.parser.remote.sitemap_loader import SitemapLoader
 from application.parser.remote.crawler_loader import CrawlerLoader
 from application.parser.remote.web_loader import WebLoader
 from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
+from application.parser.remote.github_loader import GitHubLoader
 
 
 class RemoteCreator:
@@ -10,6 +11,7 @@ class RemoteCreator:
         "sitemap": SitemapLoader,
         "crawler": CrawlerLoader,
         "reddit": RedditPostsLoaderRemote,
+        "github": GitHubLoader,
     }
 
     @classmethod

application/parser/remote/sitemap_loader.py

@@ -5,7 +5,7 @@ from application.parser.remote.base import BaseRemote
 
 class SitemapLoader(BaseRemote):
     def __init__(self, limit=20):
-        from langchain.document_loaders import WebBaseLoader
+        from langchain_community.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader
         self.limit = limit  # Adding limit to control the number of URLs to process
 

application/parser/remote/web_loader.py

@@ -1,5 +1,7 @@
 from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
 from langchain_community.document_loaders import WebBaseLoader
+from urllib.parse import urlparse
 
 headers = {
     "User-Agent": "Mozilla/5.0",
@@ -23,10 +25,20 @@ class WebLoader(BaseRemote):
             urls = [urls]
         documents = []
         for url in urls:
+            # Check if the URL scheme is provided, if not, assume http
+            if not urlparse(url).scheme:
+                url = "http://" + url
             try:
                 loader = self.loader([url], header_template=headers)
-                documents.extend(loader.load())
+                loaded_docs = loader.load()
+                for doc in loaded_docs:
+                    documents.append(
+                        Document(
+                            doc.page_content,
+                            extra_info=doc.metadata,
+                        )
+                    )
             except Exception as e:
                 print(f"Error processing URL {url}: {e}")
                 continue
-        return documents
+        return documents

application/parser/token_func.py

@@ -1,79 +0,0 @@
-import re
-from math import ceil
-from typing import List
-
-import tiktoken
-from application.parser.schema.base import Document
-
-
-def separate_header_and_body(text):
-    header_pattern = r"^(.*?\n){3}"
-    match = re.match(header_pattern, text)
-    header = match.group(0)
-    body = text[len(header):]
-    return header, body
-
-
-def group_documents(documents: List[Document], min_tokens: int, max_tokens: int) -> List[Document]:
-    docs = []
-    current_group = None
-
-    for doc in documents:
-        doc_len = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
-
-        # Check if current group is empty or if the document can be added based on token count and matching metadata
-        if (current_group is None or 
-            (len(tiktoken.get_encoding("cl100k_base").encode(current_group.text)) + doc_len < max_tokens and 
-             doc_len < min_tokens and 
-             current_group.extra_info == doc.extra_info)):
-            if current_group is None:
-                current_group = doc  # Use the document directly to retain its metadata
-            else:
-                current_group.text += " " + doc.text  # Append text to the current group
-        else:
-            docs.append(current_group)
-            current_group = doc  # Start a new group with the current document
-
-    if current_group is not None:
-        docs.append(current_group)
-
-    return docs
-
-
-def split_documents(documents: List[Document], max_tokens: int) -> List[Document]:
-    docs = []
-    for doc in documents:
-        token_length = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
-        if token_length <= max_tokens:
-            docs.append(doc)
-        else:
-            header, body = separate_header_and_body(doc.text)
-            if len(tiktoken.get_encoding("cl100k_base").encode(header)) > max_tokens:
-                body = doc.text
-                header = ""
-            num_body_parts = ceil(token_length / max_tokens)
-            part_length = ceil(len(body) / num_body_parts)
-            body_parts = [body[i:i + part_length] for i in range(0, len(body), part_length)]
-            for i, body_part in enumerate(body_parts):
-                new_doc = Document(text=header + body_part.strip(),
-                                   doc_id=f"{doc.doc_id}-{i}",
-                                   embedding=doc.embedding,
-                                   extra_info=doc.extra_info)
-                docs.append(new_doc)
-    return docs
-
-
-def group_split(documents: List[Document], max_tokens: int = 2000, min_tokens: int = 150, token_check: bool = True):
-    if not token_check:
-        return documents
-    print("Grouping small documents")
-    try:
-        documents = group_documents(documents=documents, min_tokens=min_tokens, max_tokens=max_tokens)
-    except Exception:
-        print("Grouping failed, try running without token_check")
-    print("Separating large documents")
-    try:
-        documents = split_documents(documents=documents, max_tokens=max_tokens)
-    except Exception:
-        print("Grouping failed, try running without token_check")
-    return documents

application/requirements.txt

@@ -1,34 +1,95 @@
-anthropic==0.12.0
-boto3==1.34.6
-celery==5.3.6
-dataclasses_json==0.6.3
+anthropic==0.40.0
+boto3==1.35.97
+beautifulsoup4==4.12.3
+celery==5.4.0
+dataclasses-json==0.6.7
 docx2txt==0.8
-duckduckgo-search==5.3.0
-EbookLib==0.18
-elasticsearch==8.12.0
+duckduckgo-search==6.3.0
+ebooklib==0.18
+elastic-transport==8.17.0
+elasticsearch==8.17.0
 escodegen==1.0.11
 esprima==4.0.1
-faiss-cpu==1.7.4
-Flask==3.0.1
-gunicorn==22.0.0
-html2text==2020.1.16
+esutils==1.0.1
+Flask==3.1.0
+faiss-cpu==1.9.0.post1
+flask-restx==1.3.0
+google-genai==0.5.0
+google-generativeai==0.8.3
+gTTS==2.5.4
+gunicorn==23.0.0
+html2text==2024.2.26
 javalang==0.13.0
-langchain==0.1.4
-langchain-openai==0.0.5
-openapi3_parser==1.1.16
-pandas==2.2.0
-pydantic_settings==2.1.0
-pymongo==4.6.3
-PyPDF2==3.0.1
+jinja2==3.1.5
+jiter==0.8.2
+jmespath==1.0.1
+joblib==1.4.2
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-spec==0.2.4
+jsonschema-specifications==2023.7.1
+kombu==5.4.2
+langchain==0.3.14
+langchain-community==0.3.14
+langchain-core==0.3.29
+langchain-openai==0.3.0
+langchain-text-splitters==0.3.5
+langsmith==0.2.10
+lazy-object-proxy==1.10.0
+lxml==5.3.0
+markupsafe==3.0.2
+marshmallow==3.24.1
+mpmath==1.3.0
+multidict==6.1.0
+mypy-extensions==1.0.0
+networkx==3.4.2
+numpy==2.2.1
+openai==1.59.5
+openapi-schema-validator==0.6.2
+openapi-spec-validator==0.6.0
+openapi3-parser==1.1.19
+orjson==3.10.14
+packaging==24.1
+pandas==2.2.3
+openpyxl==3.1.5
+pathable==0.4.4
+pillow==11.1.0
+portalocker==2.10.1
+prance==23.6.21.0
+primp==0.10.0
+prompt-toolkit==3.0.48
+protobuf==5.29.3
+psycopg2-binary==2.9.10
+py==1.11.0
+pydantic==2.10.4
+pydantic-core==2.27.2
+pydantic-settings==2.7.1
+pymongo==4.10.1
+pypdf2==3.0.1
+python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
-qdrant-client==1.9.0
-redis==5.0.1
-Requests==2.32.0
+python-pptx==1.0.2
+qdrant-client==1.12.2
+redis==5.2.1
+referencing==0.30.2
+regex==2024.11.6
+requests==2.32.3
 retry==0.9.2
-sentence-transformers
-tiktoken
-torch
-tqdm==4.66.3
-transformers==4.36.2
-unstructured==0.12.2
-Werkzeug==3.0.3
+sentence-transformers==3.3.1
+tiktoken==0.8.0
+tokenizers==0.21.0
+torch==2.5.1
+tqdm==4.67.1
+transformers==4.48.0
+typing-extensions==4.12.2
+typing-inspect==0.9.0
+tzdata==2024.2
+urllib3==2.3.0
+vine==5.1.0
+wcwidth==0.2.13
+werkzeug==3.1.3
+yarl==1.18.3
+markdownify==0.14.1
+tldextract==5.1.3
+websockets==14.1

application/retriever/base.py

@@ -12,3 +12,7 @@ class BaseRetriever(ABC):
     @abstractmethod
     def search(self, *args, **kwargs):
         pass
+
+    @abstractmethod
+    def get_params(self):
+        pass

application/retriever/brave_search.py

@@ -2,7 +2,6 @@ import json
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.llm.llm_creator import LLMCreator
-from application.utils import count_tokens
 from langchain_community.tools import BraveSearch
 
 
@@ -72,23 +71,13 @@ class BraveRetSearch(BaseRetriever):
         for doc in docs:
             yield {"source": doc}
 
-        if len(self.chat_history) > 1:
-            tokens_current_history = 0
-            # count tokens in history
-            self.chat_history.reverse()
+        if len(self.chat_history) > 0:
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
-                        i["response"]
+                    messages_combine.append({"role": "user", "content": i["prompt"]})
+                    messages_combine.append(
+                        {"role": "assistant", "content": i["response"]}
                     )
-                    if tokens_current_history + tokens_batch < self.token_limit:
-                        tokens_current_history += tokens_batch
-                        messages_combine.append(
-                            {"role": "user", "content": i["prompt"]}
-                        )
-                        messages_combine.append(
-                            {"role": "system", "content": i["response"]}
-                        )
         messages_combine.append({"role": "user", "content": self.question})
 
         llm = LLMCreator.create_llm(
@@ -101,3 +90,15 @@ class BraveRetSearch(BaseRetriever):
 
     def search(self):
         return self._get_data()
+
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/classic_rag.py

@@ -1,10 +1,8 @@
-import os
-from application.retriever.base import BaseRetriever
 from application.core.settings import settings
-from application.vectorstore.vector_creator import VectorCreator
-from application.llm.llm_creator import LLMCreator
+from application.retriever.base import BaseRetriever
+from application.tools.agent import Agent
 
-from application.utils import count_tokens
+from application.vectorstore.vector_creator import VectorCreator
 
 
 class ClassicRAG(BaseRetriever):
@@ -21,7 +19,7 @@ class ClassicRAG(BaseRetriever):
         user_api_key=None,
     ):
         self.question = question
-        self.vectorstore = self._get_vectorstore(source=source)
+        self.vectorstore = source["active_docs"] if "active_docs" in source else None
         self.chat_history = chat_history
         self.prompt = prompt
         self.chunks = chunks
@@ -38,21 +36,6 @@ class ClassicRAG(BaseRetriever):
         )
         self.user_api_key = user_api_key
 
-    def _get_vectorstore(self, source):
-        if "active_docs" in source:
-            if source["active_docs"].split("/")[0] == "default":
-                vectorstore = ""
-            elif source["active_docs"].split("/")[0] == "local":
-                vectorstore = "indexes/" + source["active_docs"]
-            else:
-                vectorstore = "vectors/" + source["active_docs"]
-            if source["active_docs"] == "default":
-                vectorstore = ""
-        else:
-            vectorstore = ""
-        vectorstore = os.path.join("application", vectorstore)
-        return vectorstore
-
     def _get_data(self):
         if self.chunks == 0:
             docs = []
@@ -63,11 +46,9 @@ class ClassicRAG(BaseRetriever):
             docs_temp = docsearch.search(self.question, k=self.chunks)
             docs = [
                 {
-                    "title": (
-                        i.metadata["title"].split("/")[-1]
-                        if i.metadata
-                        else i.page_content
-                    ),
+                    "title": i.metadata.get(
+                        "title", i.metadata.get("post_title", i.page_content)
+                    ).split("/")[-1],
                     "text": i.page_content,
                     "source": (
                         i.metadata.get("source")
@@ -77,8 +58,6 @@ class ClassicRAG(BaseRetriever):
                 }
                 for i in docs_temp
             ]
-        if settings.LLM_NAME == "llama.cpp":
-            docs = [docs[0]]
 
         return docs
 
@@ -92,32 +71,39 @@ class ClassicRAG(BaseRetriever):
         for doc in docs:
             yield {"source": doc}
 
-        if len(self.chat_history) > 1:
-            tokens_current_history = 0
-            # count tokens in history
-            self.chat_history.reverse()
+        if len(self.chat_history) > 0:
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
-                        i["response"]
+                    messages_combine.append({"role": "user", "content": i["prompt"]})
+                    messages_combine.append(
+                        {"role": "assistant", "content": i["response"]}
                     )
-                    if tokens_current_history + tokens_batch < self.token_limit:
-                        tokens_current_history += tokens_batch
-                        messages_combine.append(
-                            {"role": "user", "content": i["prompt"]}
-                        )
-                        messages_combine.append(
-                            {"role": "system", "content": i["response"]}
-                        )
         messages_combine.append({"role": "user", "content": self.question})
-
-        llm = LLMCreator.create_llm(
-            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        # llm = LLMCreator.create_llm(
+        #     settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        # )
+        # completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        agent = Agent(
+            llm_name=settings.LLM_NAME,
+            gpt_model=self.gpt_model,
+            api_key=settings.API_KEY,
+            user_api_key=self.user_api_key,
         )
-
-        completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        completion = agent.gen(messages_combine)
         for line in completion:
             yield {"answer": str(line)}
 
     def search(self):
         return self._get_data()
+
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.vectorstore,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key,
+        }

application/retriever/duckduck_search.py

@@ -1,7 +1,6 @@
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.llm.llm_creator import LLMCreator
-from application.utils import count_tokens
 from langchain_community.tools import DuckDuckGoSearchResults
 from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
 
@@ -89,22 +88,12 @@ class DuckDuckSearch(BaseRetriever):
         for doc in docs:
             yield {"source": doc}
 
-        if len(self.chat_history) > 1:
-            tokens_current_history = 0
-            # count tokens in history
-            self.chat_history.reverse()
+        if len(self.chat_history) > 0:      
             for i in self.chat_history:
-                if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(
-                        i["response"]
-                    )
-                    if tokens_current_history + tokens_batch < self.token_limit:
-                        tokens_current_history += tokens_batch
+                    if "prompt" in i and "response" in i:
+                        messages_combine.append({"role": "user", "content": i["prompt"]})
                         messages_combine.append(
-                            {"role": "user", "content": i["prompt"]}
-                        )
-                        messages_combine.append(
-                            {"role": "system", "content": i["response"]}
+                            {"role": "assistant", "content": i["response"]}
                         )
         messages_combine.append({"role": "user", "content": self.question})
 
@@ -118,3 +107,15 @@ class DuckDuckSearch(BaseRetriever):
 
     def search(self):
         return self._get_data()
+    
+    def get_params(self):
+        return {
+            "question": self.question,
+            "source": self.source,
+            "chat_history": self.chat_history,
+            "prompt": self.prompt,
+            "chunks": self.chunks,
+            "token_limit": self.token_limit,
+            "gpt_model": self.gpt_model,
+            "user_api_key": self.user_api_key
+        }

application/retriever/retriever_creator.py

@@ -5,15 +5,16 @@ from application.retriever.brave_search import BraveRetSearch
 
 
 class RetrieverCreator:
-    retievers = {
+    retrievers = {
         'classic': ClassicRAG,
         'duckduck_search': DuckDuckSearch,
-        'brave_search': BraveRetSearch
+        'brave_search': BraveRetSearch,
+        'default': ClassicRAG
     }
 
     @classmethod
     def create_retriever(cls, type, *args, **kwargs):
-        retiever_class = cls.retievers.get(type.lower())
+        retiever_class = cls.retrievers.get(type.lower())
         if not retiever_class:
             raise ValueError(f"No retievers class found for type {type}")
         return retiever_class(*args, **kwargs)

application/tools/agent.py

@@ -0,0 +1,160 @@
+from application.core.mongo_db import MongoDB
+from application.llm.llm_creator import LLMCreator
+from application.tools.llm_handler import get_llm_handler
+from application.tools.tool_action_parser import ToolActionParser
+from application.tools.tool_manager import ToolManager
+
+
+class Agent:
+    def __init__(self, llm_name, gpt_model, api_key, user_api_key=None):
+        # Initialize the LLM with the provided parameters
+        self.llm = LLMCreator.create_llm(
+            llm_name, api_key=api_key, user_api_key=user_api_key
+        )
+        self.llm_handler = get_llm_handler(llm_name)
+        self.gpt_model = gpt_model
+        # Static tool configuration (to be replaced later)
+        self.tools = []
+        self.tool_config = {}
+
+    def _get_user_tools(self, user="local"):
+        mongo = MongoDB.get_client()
+        db = mongo["docsgpt"]
+        user_tools_collection = db["user_tools"]
+        user_tools = user_tools_collection.find({"user": user, "status": True})
+        user_tools = list(user_tools)
+        tools_by_id = {str(tool["_id"]): tool for tool in user_tools}
+        return tools_by_id
+
+    def _build_tool_parameters(self, action):
+        params = {"type": "object", "properties": {}, "required": []}
+        for param_type in ["query_params", "headers", "body", "parameters"]:
+            if param_type in action and action[param_type].get("properties"):
+                for k, v in action[param_type]["properties"].items():
+                    if v.get("filled_by_llm", True):
+                        params["properties"][k] = {
+                            key: value
+                            for key, value in v.items()
+                            if key != "filled_by_llm" and key != "value"
+                        }
+
+                        params["required"].append(k)
+        return params
+
+    def _prepare_tools(self, tools_dict):
+        self.tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": f"{action['name']}_{tool_id}",
+                    "description": action["description"],
+                    "parameters": self._build_tool_parameters(action),
+                },
+            }
+            for tool_id, tool in tools_dict.items()
+            for action in (
+                tool["config"]["actions"].values()
+                if tool["name"] == "api_tool"
+                else tool["actions"]
+            )
+            if action.get("active", True)
+        ]
+
+    def _execute_tool_action(self, tools_dict, call):
+        parser = ToolActionParser(self.llm.__class__.__name__)
+        tool_id, action_name, call_args = parser.parse_args(call)
+
+        tool_data = tools_dict[tool_id]
+        action_data = (
+            tool_data["config"]["actions"][action_name]
+            if tool_data["name"] == "api_tool"
+            else next(
+                action
+                for action in tool_data["actions"]
+                if action["name"] == action_name
+            )
+        )
+
+        query_params, headers, body, parameters = {}, {}, {}, {}
+        param_types = {
+            "query_params": query_params,
+            "headers": headers,
+            "body": body,
+            "parameters": parameters,
+        }
+
+        for param_type, target_dict in param_types.items():
+            if param_type in action_data and action_data[param_type].get("properties"):
+                for param, details in action_data[param_type]["properties"].items():
+                    if param not in call_args and "value" in details:
+                        target_dict[param] = details["value"]
+
+        for param, value in call_args.items():
+            for param_type, target_dict in param_types.items():
+                if param_type in action_data and param in action_data[param_type].get(
+                    "properties", {}
+                ):
+                    target_dict[param] = value
+
+        tm = ToolManager(config={})
+        tool = tm.load_tool(
+            tool_data["name"],
+            tool_config=(
+                {
+                    "url": tool_data["config"]["actions"][action_name]["url"],
+                    "method": tool_data["config"]["actions"][action_name]["method"],
+                    "headers": headers,
+                    "query_params": query_params,
+                }
+                if tool_data["name"] == "api_tool"
+                else tool_data["config"]
+            ),
+        )
+        if tool_data["name"] == "api_tool":
+            print(
+                f"Executing api: {action_name} with query_params: {query_params}, headers: {headers}, body: {body}"
+            )
+            result = tool.execute_action(action_name, **body)
+        else:
+            print(f"Executing tool: {action_name} with args: {call_args}")
+            result = tool.execute_action(action_name, **parameters)
+        call_id = getattr(call, "id", None)
+        return result, call_id
+
+    def _simple_tool_agent(self, messages):
+        tools_dict = self._get_user_tools()
+        self._prepare_tools(tools_dict)
+
+        resp = self.llm.gen(model=self.gpt_model, messages=messages, tools=self.tools)
+
+        if isinstance(resp, str):
+            yield resp
+            return
+        if hasattr(resp, "message") and hasattr(resp.message, "content"):
+            yield resp.message.content
+            return
+
+        resp = self.llm_handler.handle_response(self, resp, tools_dict, messages)
+
+        if isinstance(resp, str):
+            yield resp
+        elif hasattr(resp, "message") and hasattr(resp.message, "content"):
+            yield resp.message.content
+        else:
+            completion = self.llm.gen_stream(
+                model=self.gpt_model, messages=messages, tools=self.tools
+            )
+            for line in completion:
+                yield line
+
+        return
+
+    def gen(self, messages):
+        if self.llm.supports_tools():
+            resp = self._simple_tool_agent(messages)
+            for line in resp:
+                yield line
+        else:
+            resp = self.llm.gen_stream(model=self.gpt_model, messages=messages)
+            for line in resp:
+                yield line

application/tools/base.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+
+
+class Tool(ABC):
+    @abstractmethod
+    def execute_action(self, action_name: str, **kwargs):
+        pass
+
+    @abstractmethod
+    def get_actions_metadata(self):
+        """
+        Returns a list of JSON objects describing the actions supported by the tool.
+        """
+        pass
+
+    @abstractmethod
+    def get_config_requirements(self):
+        """
+        Returns a dictionary describing the configuration requirements for the tool.
+        """
+        pass

application/tools/implementations/api_tool.py

@@ -0,0 +1,54 @@
+import json
+
+import requests
+from application.tools.base import Tool
+
+
+class APITool(Tool):
+    """
+    API Tool
+    A flexible tool for performing various API actions (e.g., sending messages, retrieving data) via custom user-specified APIs
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.url = config.get("url", "")
+        self.method = config.get("method", "GET")
+        self.headers = config.get("headers", {"Content-Type": "application/json"})
+        self.query_params = config.get("query_params", {})
+
+    def execute_action(self, action_name, **kwargs):
+        return self._make_api_call(
+            self.url, self.method, self.headers, self.query_params, kwargs
+        )
+
+    def _make_api_call(self, url, method, headers, query_params, body):
+        if query_params:
+            url = f"{url}?{requests.compat.urlencode(query_params)}"
+        if isinstance(body, dict):
+            body = json.dumps(body)
+        try:
+            print(f"Making API call: {method} {url} with body: {body}")
+            response = requests.request(method, url, headers=headers, data=body)
+            response.raise_for_status()
+            try:
+                data = response.json()
+            except ValueError:
+                data = None
+
+            return {
+                "status_code": response.status_code,
+                "data": data,
+                "message": "API call successful.",
+            }
+        except requests.exceptions.RequestException as e:
+            return {
+                "status_code": response.status_code if response else None,
+                "message": f"API call failed: {str(e)}",
+            }
+
+    def get_actions_metadata(self):
+        return []
+
+    def get_config_requirements(self):
+        return {}

application/tools/implementations/cryptoprice.py

@@ -0,0 +1,77 @@
+import requests
+from application.tools.base import Tool
+
+
+class CryptoPriceTool(Tool):
+    """
+    CryptoPrice
+    A tool for retrieving cryptocurrency prices using the CryptoCompare public API
+    """
+
+    def __init__(self, config):
+        self.config = config
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {"cryptoprice_get": self._get_price}
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _get_price(self, symbol, currency):
+        """
+        Fetches the current price of a given cryptocurrency symbol in the specified currency.
+        Example:
+            symbol = "BTC"
+            currency = "USD"
+            returns price in USD.
+        """
+        url = f"https://min-api.cryptocompare.com/data/price?fsym={symbol.upper()}&tsyms={currency.upper()}"
+        response = requests.get(url)
+        if response.status_code == 200:
+            data = response.json()
+            # data will be like {"USD": <price>} if the call is successful
+            if currency.upper() in data:
+                return {
+                    "status_code": response.status_code,
+                    "price": data[currency.upper()],
+                    "message": f"Price of {symbol.upper()} in {currency.upper()} retrieved successfully.",
+                }
+            else:
+                return {
+                    "status_code": response.status_code,
+                    "message": f"Couldn't find price for {symbol.upper()} in {currency.upper()}.",
+                }
+        else:
+            return {
+                "status_code": response.status_code,
+                "message": "Failed to retrieve price.",
+            }
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "cryptoprice_get",
+                "description": "Retrieve the price of a specified cryptocurrency in a given currency",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "symbol": {
+                            "type": "string",
+                            "description": "The cryptocurrency symbol (e.g. BTC)",
+                        },
+                        "currency": {
+                            "type": "string",
+                            "description": "The currency in which you want the price (e.g. USD)",
+                        },
+                    },
+                    "required": ["symbol", "currency"],
+                    "additionalProperties": False,
+                },
+            }
+        ]
+
+    def get_config_requirements(self):
+        # No specific configuration needed for this tool as it just queries a public endpoint
+        return {}

application/tools/implementations/postgres.py

@@ -0,0 +1,163 @@
+import psycopg2
+from application.tools.base import Tool
+
+class PostgresTool(Tool):
+    """
+    PostgreSQL Database Tool
+    A tool for connecting to a PostgreSQL database using a connection string,
+    executing SQL queries, and retrieving schema information.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.connection_string = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "postgres_execute_sql": self._execute_sql,
+            "postgres_get_schema": self._get_schema,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _execute_sql(self, sql_query):
+        """
+        Executes an SQL query against the PostgreSQL database using a connection string.
+        """
+        conn = None  # Initialize conn to None for error handling
+        try:
+            conn = psycopg2.connect(self.connection_string)
+            cur = conn.cursor()
+            cur.execute(sql_query)
+            conn.commit()
+
+            if sql_query.strip().lower().startswith("select"):
+                column_names = [desc[0] for desc in cur.description] if cur.description else []
+                results = []
+                rows = cur.fetchall()
+                for row in rows:
+                    results.append(dict(zip(column_names, row)))
+                response_data = {"data": results, "column_names": column_names}
+            else:
+                row_count = cur.rowcount
+                response_data = {"message": f"Query executed successfully, {row_count} rows affected."}
+
+            cur.close()
+            return {
+                "status_code": 200,
+                "message": "SQL query executed successfully.",
+                "response_data": response_data,
+            }
+
+        except psycopg2.Error as e:
+            error_message = f"Database error: {e}"
+            print(f"Database error: {e}")
+            return {
+                "status_code": 500,
+                "message": "Failed to execute SQL query.",
+                "error": error_message,
+            }
+        finally:
+            if conn:  # Ensure connection is closed even if errors occur
+                conn.close()
+
+    def _get_schema(self, db_name):
+        """
+        Retrieves the schema of the PostgreSQL database using a connection string.
+        """
+        conn = None # Initialize conn to None for error handling
+        try:
+            conn = psycopg2.connect(self.connection_string)
+            cur = conn.cursor()
+
+            cur.execute("""
+                SELECT
+                    table_name,
+                    column_name,
+                    data_type,
+                    column_default,
+                    is_nullable
+                FROM
+                    information_schema.columns
+                WHERE
+                    table_schema = 'public'
+                ORDER BY
+                    table_name,
+                    ordinal_position;
+            """)
+
+            schema_data = {}
+            for row in cur.fetchall():
+                table_name, column_name, data_type, column_default, is_nullable = row
+                if table_name not in schema_data:
+                    schema_data[table_name] = []
+                schema_data[table_name].append({
+                    "column_name": column_name,
+                    "data_type": data_type,
+                    "column_default": column_default,
+                    "is_nullable": is_nullable
+                })
+
+            cur.close()
+            return {
+                "status_code": 200,
+                "message": "Database schema retrieved successfully.",
+                "schema": schema_data,
+            }
+
+        except psycopg2.Error as e:
+            error_message = f"Database error: {e}"
+            print(f"Database error: {e}")
+            return {
+                "status_code": 500,
+                "message": "Failed to retrieve database schema.",
+                "error": error_message,
+            }
+        finally:
+            if conn: # Ensure connection is closed even if errors occur
+                conn.close()
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "postgres_execute_sql",
+                "description": "Execute an SQL query against the PostgreSQL database and return the results. Use this tool to interact with the database, e.g., retrieve specific data or perform updates. Only SELECT queries will return data, other queries will return execution status.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "sql_query": {
+                            "type": "string",
+                            "description": "The SQL query to execute.",
+                        },
+                    },
+                    "required": ["sql_query"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "postgres_get_schema",
+                "description": "Retrieve the schema of the PostgreSQL database, including tables and their columns. Use this to understand the database structure before executing queries. db_name is 'default' if not provided.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "db_name": {
+                            "type": "string",
+                            "description": "The name of the database to retrieve the schema for.",
+                        },
+                    },
+                    "required": ["db_name"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {
+                "type": "string",
+                "description": "PostgreSQL database connection string (e.g., 'postgresql://user:password@host:port/dbname')",
+            },
+        }

application/tools/implementations/telegram.py

@@ -0,0 +1,86 @@
+import requests
+from application.tools.base import Tool
+
+
+class TelegramTool(Tool):
+    """
+    Telegram Bot
+    A flexible Telegram tool for performing various actions (e.g., sending messages, images).
+    Requires a bot token and chat ID for configuration
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.token = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "telegram_send_message": self._send_message,
+            "telegram_send_image": self._send_image,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _send_message(self, text, chat_id):
+        print(f"Sending message: {text}")
+        url = f"https://api.telegram.org/bot{self.token}/sendMessage"
+        payload = {"chat_id": chat_id, "text": text}
+        response = requests.post(url, data=payload)
+        return {"status_code": response.status_code, "message": "Message sent"}
+
+    def _send_image(self, image_url, chat_id):
+        print(f"Sending image: {image_url}")
+        url = f"https://api.telegram.org/bot{self.token}/sendPhoto"
+        payload = {"chat_id": chat_id, "photo": image_url}
+        response = requests.post(url, data=payload)
+        return {"status_code": response.status_code, "message": "Image sent"}
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "telegram_send_message",
+                "description": "Send a notification to Telegram chat",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "text": {
+                            "type": "string",
+                            "description": "Text to send in the notification",
+                        },
+                        "chat_id": {
+                            "type": "string",
+                            "description": "Chat ID to send the notification to",
+                        },
+                    },
+                    "required": ["text"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "telegram_send_image",
+                "description": "Send an image to the Telegram chat",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "image_url": {
+                            "type": "string",
+                            "description": "URL of the image to send",
+                        },
+                        "chat_id": {
+                            "type": "string",
+                            "description": "Chat ID to send the image to",
+                        },
+                    },
+                    "required": ["image_url"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {"type": "string", "description": "Bot token for authentication"},
+        }

application/tools/llm_handler.py

@@ -0,0 +1,97 @@
+import json
+from abc import ABC, abstractmethod
+
+
+class LLMHandler(ABC):
+    @abstractmethod
+    def handle_response(self, agent, resp, tools_dict, messages, **kwargs):
+        pass
+
+
+class OpenAILLMHandler(LLMHandler):
+    def handle_response(self, agent, resp, tools_dict, messages):
+        while resp.finish_reason == "tool_calls":
+            message = json.loads(resp.model_dump_json())["message"]
+            keys_to_remove = {"audio", "function_call", "refusal"}
+            filtered_data = {
+                k: v for k, v in message.items() if k not in keys_to_remove
+            }
+            messages.append(filtered_data)
+
+            tool_calls = resp.message.tool_calls
+            for call in tool_calls:
+                try:
+                    tool_response, call_id = agent._execute_tool_action(
+                        tools_dict, call
+                    )
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "content": str(tool_response),
+                            "tool_call_id": call_id,
+                        }
+                    )
+                except Exception as e:
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "content": f"Error executing tool: {str(e)}",
+                            "tool_call_id": call_id,
+                        }
+                    )
+            resp = agent.llm.gen(
+                model=agent.gpt_model, messages=messages, tools=agent.tools
+            )
+        return resp
+
+
+class GoogleLLMHandler(LLMHandler):
+    def handle_response(self, agent, resp, tools_dict, messages):
+        from google.genai import types
+
+        while True:
+            response = agent.llm.gen(
+                model=agent.gpt_model, messages=messages, tools=agent.tools
+            )
+            if response.candidates and response.candidates[0].content.parts:
+                tool_call_found = False
+                for part in response.candidates[0].content.parts:
+                    if part.function_call:
+                        tool_call_found = True
+                        tool_response, call_id = agent._execute_tool_action(
+                            tools_dict, part.function_call
+                        )
+                        function_response_part = types.Part.from_function_response(
+                            name=part.function_call.name,
+                            response={"result": tool_response},
+                        )
+
+                        messages.append(
+                            {"role": "model", "content": [part.to_json_dict()]}
+                        )
+                        messages.append(
+                            {
+                                "role": "tool",
+                                "content": [function_response_part.to_json_dict()],
+                            }
+                        )
+
+                if (
+                    not tool_call_found
+                    and response.candidates[0].content.parts
+                    and response.candidates[0].content.parts[0].text
+                ):
+                    return response.candidates[0].content.parts[0].text
+                elif not tool_call_found:
+                    return response.candidates[0].content.parts
+
+            else:
+                return response
+
+
+def get_llm_handler(llm_type):
+    handlers = {
+        "openai": OpenAILLMHandler(),
+        "google": GoogleLLMHandler(),
+    }
+    return handlers.get(llm_type, OpenAILLMHandler())

application/tools/tool_action_parser.py

@@ -0,0 +1,26 @@
+import json
+
+
+class ToolActionParser:
+    def __init__(self, llm_type):
+        self.llm_type = llm_type
+        self.parsers = {
+            "OpenAILLM": self._parse_openai_llm,
+            "GoogleLLM": self._parse_google_llm,
+        }
+
+    def parse_args(self, call):
+        parser = self.parsers.get(self.llm_type, self._parse_openai_llm)
+        return parser(call)
+
+    def _parse_openai_llm(self, call):
+        call_args = json.loads(call.function.arguments)
+        tool_id = call.function.name.split("_")[-1]
+        action_name = call.function.name.rsplit("_", 1)[0]
+        return tool_id, action_name, call_args
+
+    def _parse_google_llm(self, call):
+        call_args = call.args
+        tool_id = call.name.split("_")[-1]
+        action_name = call.name.rsplit("_", 1)[0]
+        return tool_id, action_name, call_args

application/tools/tool_manager.py

@@ -0,0 +1,46 @@
+import importlib
+import inspect
+import os
+import pkgutil
+
+from application.tools.base import Tool
+
+
+class ToolManager:
+    def __init__(self, config):
+        self.config = config
+        self.tools = {}
+        self.load_tools()
+
+    def load_tools(self):
+        tools_dir = os.path.join(os.path.dirname(__file__), "implementations")
+        for finder, name, ispkg in pkgutil.iter_modules([tools_dir]):
+            if name == "base" or name.startswith("__"):
+                continue
+            module = importlib.import_module(
+                f"application.tools.implementations.{name}"
+            )
+            for member_name, obj in inspect.getmembers(module, inspect.isclass):
+                if issubclass(obj, Tool) and obj is not Tool:
+                    tool_config = self.config.get(name, {})
+                    self.tools[name] = obj(tool_config)
+
+    def load_tool(self, tool_name, tool_config):
+        self.config[tool_name] = tool_config
+        module = importlib.import_module(
+            f"application.tools.implementations.{tool_name}"
+        )
+        for member_name, obj in inspect.getmembers(module, inspect.isclass):
+            if issubclass(obj, Tool) and obj is not Tool:
+                return obj(tool_config)
+
+    def execute_action(self, tool_name, action_name, **kwargs):
+        if tool_name not in self.tools:
+            raise ValueError(f"Tool '{tool_name}' not loaded")
+        return self.tools[tool_name].execute_action(action_name, **kwargs)
+
+    def get_all_actions_metadata(self):
+        metadata = []
+        for tool in self.tools.values():
+            metadata.extend(tool.get_actions_metadata())
+        return metadata

application/tts/base.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+
+
+class BaseTTS(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def text_to_speech(self, *args, **kwargs):
+        pass

application/tts/elevenlabs.py

@@ -0,0 +1,84 @@
+import asyncio
+import websockets
+import json
+import base64
+from io import BytesIO
+from application.tts.base import BaseTTS
+
+
+class ElevenlabsTTS(BaseTTS):
+    def __init__(self):        
+        self.api_key = 'ELEVENLABS_API_KEY'# here you should put your api key
+        self.model = "eleven_flash_v2_5"
+        self.voice = "VOICE_ID" # this is the hash code for the voice not the name!
+        self.write_audio = 1
+
+    def text_to_speech(self, text):
+        asyncio.run(self._text_to_speech_websocket(text))
+
+    async def _text_to_speech_websocket(self, text):
+        uri = f"wss://api.elevenlabs.io/v1/text-to-speech/{self.voice}/stream-input?model_id={self.model}"
+        websocket = await websockets.connect(uri)
+        payload = {
+            "text": " ",
+            "voice_settings": {
+                "stability": 0.5,
+                "similarity_boost": 0.8,
+            },
+            "xi_api_key": self.api_key,
+        }
+
+        await websocket.send(json.dumps(payload))
+        
+        async def listen():
+            while 1:
+                try:
+                    msg = await websocket.recv()
+                    data = json.loads(msg)
+
+                    if data.get("audio"):
+                        print("audio received")
+                        yield base64.b64decode(data["audio"])
+                    elif data.get("isFinal"):
+                        break
+                except websockets.exceptions.ConnectionClosed:
+                    print("websocket closed")
+                    break
+        listen_task =  asyncio.create_task(self.stream(listen()))
+        
+        await websocket.send(json.dumps({"text": text}))
+        # this is to signal the end of the text, either use this or flush
+        await websocket.send(json.dumps({"text": ""})) 
+
+        await listen_task
+    
+    async def stream(self, audio_stream):
+        if self.write_audio:
+            audio_bytes = BytesIO()
+            async for chunk in audio_stream:
+                if chunk:
+                    audio_bytes.write(chunk)
+            with open("output_audio.mp3", "wb") as f:
+                f.write(audio_bytes.getvalue())
+        
+        else:
+            async for chunk in audio_stream:
+                pass # depends on the streamer!
+
+
+def test_elevenlabs_websocket():
+    """
+    Tests the ElevenlabsTTS text_to_speech method with a sample prompt.
+    Prints out the base64-encoded result and writes it to 'output_audio.mp3'.
+    """
+    # Instantiate your TTS class
+    tts = ElevenlabsTTS()
+
+    # Call the method with some sample text
+    tts.text_to_speech("Hello from ElevenLabs WebSocket!")
+
+    print("Saved audio to output_audio.mp3.")
+
+
+if __name__ == "__main__":
+    test_elevenlabs_websocket()

application/tts/google_tts.py

@@ -0,0 +1,19 @@
+import io
+import base64
+from gtts import gTTS
+from application.tts.base import BaseTTS
+
+
+class GoogleTTS(BaseTTS):
+    def __init__(self):
+        pass
+
+
+    def text_to_speech(self, text):
+        lang = "en"
+        audio_fp = io.BytesIO()
+        tts = gTTS(text=text, lang=lang, slow=False)
+        tts.write_to_fp(audio_fp)
+        audio_fp.seek(0)
+        audio_base64 = base64.b64encode(audio_fp.read()).decode("utf-8")
+        return audio_base64, lang

application/usage.py

@@ -1,10 +1,9 @@
 import sys
-from pymongo import MongoClient
 from datetime import datetime
-from application.core.settings import settings
-from application.utils import count_tokens
+from application.core.mongo_db import MongoDB
+from application.utils import num_tokens_from_string, num_tokens_from_object_or_list
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 usage_collection = db["token_usage"]
 
@@ -22,11 +21,16 @@ def update_token_usage(user_api_key, token_usage):
 
 
 def gen_token_usage(func):
-    def wrapper(self, model, messages, stream, **kwargs):
+    def wrapper(self, model, messages, stream, tools, **kwargs):
         for message in messages:
-            self.token_usage["prompt_tokens"] += count_tokens(message["content"])
-        result = func(self, model, messages, stream, **kwargs)
-        self.token_usage["generated_tokens"] += count_tokens(result)
+            if message["content"]:
+                self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
+        result = func(self, model, messages, stream, tools, **kwargs)
+        # check if result is a string
+        if isinstance(result, str):
+            self.token_usage["generated_tokens"] += num_tokens_from_string(result)
+        else:
+            self.token_usage["generated_tokens"] += num_tokens_from_object_or_list(result)
         update_token_usage(self.user_api_key, self.token_usage)
         return result
 
@@ -34,16 +38,16 @@ def gen_token_usage(func):
 
 
 def stream_token_usage(func):
-    def wrapper(self, model, messages, stream, **kwargs):
+    def wrapper(self, model, messages, stream, tools, **kwargs):
         for message in messages:
-            self.token_usage["prompt_tokens"] += count_tokens(message["content"])
+            self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
         batch = []
-        result = func(self, model, messages, stream, **kwargs)
+        result = func(self, model, messages, stream, tools, **kwargs)
         for r in result:
             batch.append(r)
             yield r
         for line in batch:
-            self.token_usage["generated_tokens"] += count_tokens(line)
+            self.token_usage["generated_tokens"] += num_tokens_from_string(line)
         update_token_usage(self.user_api_key, self.token_usage)
 
     return wrapper

application/utils.py

@@ -1,6 +1,97 @@
-from transformers import GPT2TokenizerFast
+import tiktoken
+import hashlib
+from flask import jsonify, make_response
 
-tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
-tokenizer.model_max_length = 100000
-def count_tokens(string):
-    return len(tokenizer(string)['input_ids'])
+
+_encoding = None
+
+
+def get_encoding():
+    global _encoding
+    if _encoding is None:
+        _encoding = tiktoken.get_encoding("cl100k_base")
+    return _encoding
+
+
+def num_tokens_from_string(string: str) -> int:
+    encoding = get_encoding()
+    if isinstance(string, str):
+        num_tokens = len(encoding.encode(string))
+        return num_tokens
+    else:
+        return 0
+
+def num_tokens_from_object_or_list(thing):
+    if isinstance(thing, list):
+        return sum([num_tokens_from_object_or_list(x) for x in thing])
+    elif isinstance(thing, dict):
+        return sum([num_tokens_from_object_or_list(x) for x in thing.values()])
+    elif isinstance(thing, str):
+        return num_tokens_from_string(thing)
+    else:
+        return 0
+
+def count_tokens_docs(docs):
+    docs_content = ""
+    for doc in docs:
+        docs_content += doc.page_content
+
+    tokens = num_tokens_from_string(docs_content)
+    return tokens
+
+
+def check_required_fields(data, required_fields):
+    missing_fields = [field for field in required_fields if field not in data]
+    if missing_fields:
+        return make_response(
+            jsonify(
+                {
+                    "success": False,
+                    "message": f"Missing fields: {', '.join(missing_fields)}",
+                }
+            ),
+            400,
+        )
+    return None
+
+
+def get_hash(data):
+    return hashlib.md5(data.encode()).hexdigest()
+
+def limit_chat_history(history, max_token_limit=None, gpt_model="docsgpt"):
+    """
+    Limits chat history based on token count.
+    Returns a list of messages that fit within the token limit.
+    """
+    from application.core.settings import settings
+
+    max_token_limit = (
+            max_token_limit
+            if max_token_limit and 
+            max_token_limit < settings.MODEL_TOKEN_LIMITS.get(
+                gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+            else settings.MODEL_TOKEN_LIMITS.get(
+                gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+        )
+    
+
+    if not history:
+        return []
+        
+    tokens_current_history = 0
+    trimmed_history = []
+    
+    for message in reversed(history):
+        if "prompt" in message and "response" in message:
+            tokens_batch = num_tokens_from_string(message["prompt"]) + num_tokens_from_string(
+                message["response"]
+            )
+            if tokens_current_history + tokens_batch < max_token_limit:
+                tokens_current_history += tokens_batch
+                trimmed_history.insert(0, message)
+            else:
+                break
+
+    return trimmed_history

application/vectorstore/base.py

@@ -1,13 +1,30 @@
 from abc import ABC, abstractmethod
 import os
-from langchain_community.embeddings import (
-    HuggingFaceEmbeddings,
-    CohereEmbeddings,
-    HuggingFaceInstructEmbeddings,
-)
+from sentence_transformers import SentenceTransformer
 from langchain_openai import OpenAIEmbeddings
 from application.core.settings import settings
 
+class EmbeddingsWrapper:
+    def __init__(self, model_name, *args, **kwargs):
+        self.model = SentenceTransformer(model_name, config_kwargs={'allow_dangerous_deserialization': True}, *args, **kwargs)
+        self.dimension = self.model.get_sentence_embedding_dimension()
+
+    def embed_query(self, query: str):
+        return self.model.encode(query).tolist()
+    
+    def embed_documents(self, documents: list):
+        return self.model.encode(documents).tolist()
+    
+    def __call__(self, text):
+        if isinstance(text, str):
+            return self.embed_query(text)
+        elif isinstance(text, list):
+            return self.embed_documents(text)
+        else:
+            raise ValueError("Input must be a string or a list of strings")
+
+
+
 class EmbeddingsSingleton:
     _instances = {}
 
@@ -23,16 +40,15 @@ class EmbeddingsSingleton:
     def _create_instance(embeddings_name, *args, **kwargs):
         embeddings_factory = {
             "openai_text-embedding-ada-002": OpenAIEmbeddings,
-            "huggingface_sentence-transformers/all-mpnet-base-v2": HuggingFaceEmbeddings,
-            "huggingface_sentence-transformers-all-mpnet-base-v2": HuggingFaceEmbeddings,
-            "huggingface_hkunlp/instructor-large": HuggingFaceInstructEmbeddings,
-            "cohere_medium": CohereEmbeddings
+            "huggingface_sentence-transformers/all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
+            "huggingface_sentence-transformers-all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
+            "huggingface_hkunlp/instructor-large": lambda: EmbeddingsWrapper("hkunlp/instructor-large"),
         }
 
-        if embeddings_name not in embeddings_factory:
-            raise ValueError(f"Invalid embeddings_name: {embeddings_name}")
-
-        return embeddings_factory[embeddings_name](*args, **kwargs)
+        if embeddings_name in embeddings_factory:
+            return embeddings_factory[embeddings_name](*args, **kwargs)
+        else:
+            return EmbeddingsWrapper(embeddings_name, *args, **kwargs)
 
 class BaseVectorStore(ABC):
     def __init__(self):
@@ -58,22 +74,14 @@ class BaseVectorStore(ABC):
                     embeddings_name,
                     openai_api_key=embeddings_key
                 )
-        elif embeddings_name == "cohere_medium":
-            embedding_instance = EmbeddingsSingleton.get_instance(
-                embeddings_name,
-                cohere_api_key=embeddings_key
-            )
         elif embeddings_name == "huggingface_sentence-transformers/all-mpnet-base-v2":
             if os.path.exists("./model/all-mpnet-base-v2"):
                 embedding_instance = EmbeddingsSingleton.get_instance(
-                    embeddings_name,
-                    model_name="./model/all-mpnet-base-v2",
-                    model_kwargs={"device": "cpu"}
+                    embeddings_name="./model/all-mpnet-base-v2",
                 )
             else:
                 embedding_instance = EmbeddingsSingleton.get_instance(
                     embeddings_name,
-                    model_kwargs={"device": "cpu"}
                 )
         else:
             embedding_instance = EmbeddingsSingleton.get_instance(embeddings_name)

application/vectorstore/elasticsearch.py

@@ -9,9 +9,9 @@ import elasticsearch
 class ElasticsearchStore(BaseVectorStore):
     _es_connection = None  # Class attribute to hold the Elasticsearch connection
 
-    def __init__(self, path, embeddings_key, index_name=settings.ELASTIC_INDEX):
+    def __init__(self, source_id, embeddings_key, index_name=settings.ELASTIC_INDEX):
         super().__init__()
-        self.path = path.replace("application/indexes/", "").rstrip("/")
+        self.source_id = source_id.replace("application/indexes/", "").rstrip("/")
         self.embeddings_key = embeddings_key
         self.index_name = index_name
         
@@ -81,7 +81,7 @@ class ElasticsearchStore(BaseVectorStore):
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
         vector = embeddings.embed_query(question)
         knn = {
-            "filter": [{"match": {"metadata.store.keyword": self.path}}],
+            "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
             "field": "vector",
             "k": k,
             "num_candidates": 100,
@@ -100,7 +100,7 @@ class ElasticsearchStore(BaseVectorStore):
                             }
                         }
                     ],
-                    "filter": [{"match": {"metadata.store.keyword": self.path}}],
+                    "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
                 }
             },
             "rank": {"rrf": {}},
@@ -209,5 +209,4 @@ class ElasticsearchStore(BaseVectorStore):
 
     def delete_index(self):
         self._es_connection.delete_by_query(index=self.index_name, query={"match": {
-                                      "metadata.store.keyword": self.path}},)
-
+                                      "metadata.source_id.keyword": self.source_id}},)

application/vectorstore/faiss.py

@@ -1,21 +1,29 @@
 from langchain_community.vectorstores import FAISS
 from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
+import os
+
+def get_vectorstore(path: str) -> str:
+    if path:
+        vectorstore = os.path.join("application", "indexes", path)
+    else:
+        vectorstore = os.path.join("application")
+    return vectorstore
 
 class FaissStore(BaseVectorStore):
-
-    def __init__(self, path, embeddings_key, docs_init=None):
+    def __init__(self, source_id: str, embeddings_key: str, docs_init=None):
         super().__init__()
-        self.path = path
+        self.path = get_vectorstore(source_id)
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
-        if docs_init:
-            self.docsearch = FAISS.from_documents(
-                docs_init, embeddings
-            )
-        else:
-            self.docsearch = FAISS.load_local(
-                self.path, embeddings
-            )
+
+        try:
+            if docs_init:
+                self.docsearch = FAISS.from_documents(docs_init, embeddings)
+            else:
+                self.docsearch = FAISS.load_local(self.path, embeddings, allow_dangerous_deserialization=True)
+        except Exception:
+            raise
+
         self.assert_embedding_dimensions(embeddings)
 
     def search(self, *args, **kwargs):
@@ -31,16 +39,12 @@ class FaissStore(BaseVectorStore):
         return self.docsearch.delete(*args, **kwargs)
 
     def assert_embedding_dimensions(self, embeddings):
-        """
-        Check that the word embedding dimension of the docsearch index matches
-        the dimension of the word embeddings used 
-        """
+        """Check that the word embedding dimension of the docsearch index matches the dimension of the word embeddings used."""
         if settings.EMBEDDINGS_NAME == "huggingface_sentence-transformers/all-mpnet-base-v2":
-            try:
-                word_embedding_dimension = embeddings.client[1].word_embedding_dimension
-            except AttributeError as e:
-                raise AttributeError("word_embedding_dimension not found in embeddings.client[1]") from e
+            word_embedding_dimension = getattr(embeddings, 'dimension', None)
+            if word_embedding_dimension is None:
+                raise AttributeError("'dimension' attribute not found in embeddings instance.")
+            
             docsearch_index_dimension = self.docsearch.index.d
             if word_embedding_dimension != docsearch_index_dimension:
-                raise ValueError(f"word_embedding_dimension ({word_embedding_dimension}) " +
-                                 f"!= docsearch_index_word_embedding_dimension ({docsearch_index_dimension})")
+                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) != docsearch index dimension ({docsearch_index_dimension})")

application/vectorstore/lancedb.py

@@ -0,0 +1,119 @@
+from typing import List, Optional
+import importlib
+from application.vectorstore.base import BaseVectorStore
+from application.core.settings import settings
+
+class LanceDBVectorStore(BaseVectorStore):
+    """Class for LanceDB Vector Store integration."""
+
+    def __init__(self, path: str = settings.LANCEDB_PATH,
+                 table_name_prefix: str = settings.LANCEDB_TABLE_NAME,
+                 source_id: str = None,
+                 embeddings_key: str = "embeddings"):
+        """Initialize the LanceDB vector store."""
+        super().__init__()
+        self.path = path
+        self.table_name = f"{table_name_prefix}_{source_id}" if source_id else table_name_prefix
+        self.embeddings_key = embeddings_key
+        self._lance_db = None
+        self.docsearch = None
+        self._pa = None  # PyArrow (pa) will be lazy loaded
+
+    @property
+    def pa(self):
+        """Lazy load pyarrow module."""
+        if self._pa is None:
+            self._pa = importlib.import_module("pyarrow")
+        return self._pa
+
+    @property
+    def lancedb(self):
+        """Lazy load lancedb module."""
+        if not hasattr(self, "_lancedb_module"):
+            self._lancedb_module = importlib.import_module("lancedb")
+        return self._lancedb_module
+
+    @property
+    def lance_db(self):
+        """Lazy load the LanceDB connection."""
+        if self._lance_db is None:
+            self._lance_db = self.lancedb.connect(self.path)
+        return self._lance_db
+
+    @property
+    def table(self):
+        """Lazy load the LanceDB table."""
+        if self.docsearch is None:
+            if self.table_name in self.lance_db.table_names():
+                self.docsearch = self.lance_db.open_table(self.table_name)
+            else:
+                self.docsearch = None
+        return self.docsearch
+
+    def ensure_table_exists(self):
+        """Ensure the table exists before performing operations."""
+        if self.table is None:
+            embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
+            schema = self.pa.schema([
+                self.pa.field("vector", self.pa.list_(self.pa.float32(), list_size=embeddings.dimension)),
+                self.pa.field("text", self.pa.string()),
+                self.pa.field("metadata", self.pa.struct([
+                    self.pa.field("key", self.pa.string()),
+                    self.pa.field("value", self.pa.string())
+                ]))
+            ])
+            self.docsearch = self.lance_db.create_table(self.table_name, schema=schema)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]] = None, source_id: str = None):
+        """Add texts with metadata and their embeddings to the LanceDB table."""
+        embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_documents(texts)
+        vectors = []
+        for embedding, text, metadata in zip(embeddings, texts, metadatas or [{}] * len(texts)):
+            if source_id:
+                metadata["source_id"] = source_id
+            metadata_struct = [{"key": k, "value": str(v)} for k, v in metadata.items()]
+            vectors.append({
+                "vector": embedding,
+                "text": text,
+                "metadata": metadata_struct
+            })
+        self.ensure_table_exists()
+        self.docsearch.add(vectors)
+
+    def search(self, query: str, k: int = 2, *args, **kwargs):
+        """Search LanceDB for the top k most similar vectors."""
+        self.ensure_table_exists()
+        query_embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_query(query)
+        results = self.docsearch.search(query_embedding).limit(k).to_list()
+        return [(result["_distance"], result["text"], result["metadata"]) for result in results]
+
+    def delete_index(self):
+        """Delete the entire LanceDB index (table)."""
+        if self.table:
+            self.lance_db.drop_table(self.table_name)
+
+    def assert_embedding_dimensions(self, embeddings):
+        """Ensure that embedding dimensions match the table index dimensions."""
+        word_embedding_dimension = embeddings.dimension
+        if self.table:
+            table_index_dimension = len(self.docsearch.schema["vector"].type.value_type)
+            if word_embedding_dimension != table_index_dimension:
+                raise ValueError(
+                    f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) "
+                    f"!= table index dimension ({table_index_dimension})"
+                )
+
+    def filter_documents(self, filter_condition: dict) -> List[dict]:
+        """Filter documents based on certain conditions."""
+        self.ensure_table_exists()
+
+        # Ensure source_id exists in the filter condition
+        if 'source_id' not in filter_condition:
+            raise ValueError("filter_condition must contain 'source_id'")
+
+        source_id = filter_condition["source_id"]
+
+        # Use LanceDB's native filtering if supported, otherwise filter manually
+        filtered_data = self.docsearch.filter(lambda x: x.metadata and x.metadata.get("source_id") == source_id).to_list()
+
+        return filtered_data

application/vectorstore/milvus.py

@@ -0,0 +1,38 @@
+from typing import List, Optional
+from uuid import uuid4
+
+
+from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
+
+
+class MilvusStore(BaseVectorStore):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
+        super().__init__()
+        from langchain_milvus import Milvus
+
+        connection_args = {
+            "uri": settings.MILVUS_URI,
+            "token": settings.MILVUS_TOKEN,
+        }
+        self._docsearch = Milvus(
+            embedding_function=self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key),
+            collection_name=settings.MILVUS_COLLECTION_NAME,
+            connection_args=connection_args,
+        )
+        self._source_id = source_id
+
+    def search(self, question, k=2, *args, **kwargs):
+        expr = f"source_id == '{self._source_id}'"
+        return self._docsearch.similarity_search(query=question, k=k, expr=expr, *args, **kwargs)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]], *args, **kwargs):
+        ids = [str(uuid4()) for _ in range(len(texts))]
+
+        return self._docsearch.add_texts(texts=texts, metadatas=metadatas, ids=ids, *args, **kwargs)
+
+    def save_local(self, *args, **kwargs):
+        pass
+
+    def delete_index(self, *args, **kwargs):
+        pass

application/vectorstore/mongodb.py

@@ -1,11 +1,12 @@
-from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
 from application.vectorstore.document_class import Document
 
+
 class MongoDBVectorStore(BaseVectorStore):
     def __init__(
         self,
-        path: str = "",
+        source_id: str = "",
         embeddings_key: str = "embeddings",
         collection: str = "documents",
         index_name: str = "vector_search_index",
@@ -18,7 +19,7 @@ class MongoDBVectorStore(BaseVectorStore):
         self._embedding_key = embedding_key
         self._embeddings_key = embeddings_key
         self._mongo_uri = settings.MONGO_URI
-        self._path = path.replace("application/indexes/", "").rstrip("/")
+        self._source_id = source_id.replace("application/indexes/", "").rstrip("/")
         self._embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
 
         try:
@@ -33,27 +34,24 @@ class MongoDBVectorStore(BaseVectorStore):
         self._database = self._client[database]
         self._collection = self._database[collection]
 
-        
     def search(self, question, k=2, *args, **kwargs):
         query_vector = self._embedding.embed_query(question)
 
         pipeline = [
             {
                 "$vectorSearch": {
-                    "queryVector": query_vector, 
+                    "queryVector": query_vector,
                     "path": self._embedding_key,
-                    "limit": k, 
-                    "numCandidates": k * 10, 
+                    "limit": k,
+                    "numCandidates": k * 10,
                     "index": self._index_name,
-                    "filter": {
-                        "store": {"$eq": self._path}
-                    }
+                    "filter": {"source_id": {"$eq": self._source_id}},
                 }
             }
         ]
 
         cursor = self._collection.aggregate(pipeline)
-        
+
         results = []
         for doc in cursor:
             text = doc[self._text_key]
@@ -63,30 +61,32 @@ class MongoDBVectorStore(BaseVectorStore):
             metadata = doc
             results.append(Document(text, metadata))
         return results
-    
+
     def _insert_texts(self, texts, metadatas):
         if not texts:
             return []
         embeddings = self._embedding.embed_documents(texts)
+
         to_insert = [
             {self._text_key: t, self._embedding_key: embedding, **m}
             for t, m, embedding in zip(texts, metadatas, embeddings)
         ]
-        # insert the documents in MongoDB Atlas
+
         insert_result = self._collection.insert_many(to_insert)
         return insert_result.inserted_ids
-    
-    def add_texts(self,
+
+    def add_texts(
+        self,
         texts,
-        metadatas = None,
-        ids = None,
-        refresh_indices = True,
-        create_index_if_not_exists = True,
-        bulk_kwargs = None,
-        **kwargs,):
+        metadatas=None,
+        ids=None,
+        refresh_indices=True,
+        create_index_if_not_exists=True,
+        bulk_kwargs=None,
+        **kwargs,
+    ):
 
-
-        #dims = self._embedding.client[1].word_embedding_dimension
+        # dims = self._embedding.client[1].word_embedding_dimension
         # # check if index exists
         # if create_index_if_not_exists:
         #     # check if index exists
@@ -121,6 +121,6 @@ class MongoDBVectorStore(BaseVectorStore):
         if texts_batch:
             result_ids.extend(self._insert_texts(texts_batch, metadatas_batch))
         return result_ids
-    
+
     def delete_index(self, *args, **kwargs):
-        self._collection.delete_many({"store": self._path})
+        self._collection.delete_many({"source_id": self._source_id})

application/vectorstore/qdrant.py

@@ -5,12 +5,12 @@ from qdrant_client import models
 
 
 class QdrantStore(BaseVectorStore):
-    def __init__(self, path: str = "", embeddings_key: str = "embeddings"):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
         self._filter = models.Filter(
             must=[
                 models.FieldCondition(
-                    key="metadata.store",
-                    match=models.MatchValue(value=path.replace("application/indexes/", "").rstrip("/")),
+                    key="metadata.source_id",
+                    match=models.MatchValue(value=source_id.replace("application/indexes/", "").rstrip("/")),
                 )
             ]
         )

application/vectorstore/vector_creator.py

@@ -1,5 +1,6 @@
 from application.vectorstore.faiss import FaissStore
 from application.vectorstore.elasticsearch import ElasticsearchStore
+from application.vectorstore.milvus import MilvusStore
 from application.vectorstore.mongodb import MongoDBVectorStore
 from application.vectorstore.qdrant import QdrantStore
 
@@ -10,6 +11,7 @@ class VectorCreator:
         "elasticsearch": ElasticsearchStore,
         "mongodb": MongoDBVectorStore,
         "qdrant": QdrantStore,
+        "milvus": MilvusStore,
     }
 
     @classmethod

application/worker.py

@@ -1,35 +1,44 @@
+import logging
 import os
 import shutil
 import string
 import zipfile
-import tiktoken
+from collections import Counter
 from urllib.parse import urljoin
 
 import requests
+from bson.objectid import ObjectId
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 from application.parser.file.bulk import SimpleDirectoryReader
+from application.parser.embedding_pipeline import embed_and_store_documents
 from application.parser.remote.remote_creator import RemoteCreator
-from application.parser.open_ai_func import call_openai_api
 from application.parser.schema.base import Document
-from application.parser.token_func import group_split
+from application.parser.chunking import Chunker
+from application.utils import count_tokens_docs
+
+mongo = MongoDB.get_client()
+db = mongo["docsgpt"]
+sources_collection = db["sources"]
+
+# Constants
+MIN_TOKENS = 150
+MAX_TOKENS = 1250
+RECURSION_DEPTH = 2
 
 # Define a function to extract metadata from a given filename.
 def metadata_from_filename(title):
-    store = "/".join(title.split("/")[1:3])
-    return {"title": title, "store": store}
-
+    return {"title": title}
 
 # Define a function to generate a random string of a given length.
 def generate_random_string(length):
     return "".join([string.ascii_letters[i % 52] for i in range(length)])
 
-
 current_dir = os.path.dirname(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 )
 
-
 def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
     """
     Recursively extract zip files with a limit on recursion depth.
@@ -41,12 +50,16 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
         max_depth (int): Maximum allowed depth of recursion to prevent infinite loops.
     """
     if current_depth > max_depth:
-        print(f"Reached maximum recursion depth of {max_depth}")
+        logging.warning(f"Reached maximum recursion depth of {max_depth}")
         return
 
-    with zipfile.ZipFile(zip_path, "r") as zip_ref:
-        zip_ref.extractall(extract_to)
-    os.remove(zip_path)  # Remove the zip file after extracting
+    try:
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(extract_to)
+        os.remove(zip_path)  # Remove the zip file after extracting
+    except Exception as e:
+        logging.error(f"Error extracting zip file {zip_path}: {e}")
+        return
 
     # Check for nested zip files and extract them
     for root, dirs, files in os.walk(extract_to):
@@ -56,9 +69,43 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
                 file_path = os.path.join(root, file)
                 extract_zip_recursive(file_path, root, current_depth + 1, max_depth)
 
+def download_file(url, params, dest_path):
+    try:
+        response = requests.get(url, params=params)
+        response.raise_for_status()
+        with open(dest_path, "wb") as f:
+            f.write(response.content)
+    except requests.RequestException as e:
+        logging.error(f"Error downloading file: {e}")
+        raise
+
+def upload_index(full_path, file_data):
+    try:
+        if settings.VECTOR_STORE == "faiss":
+            files = {
+                "file_faiss": open(full_path + "/index.faiss", "rb"),
+                "file_pkl": open(full_path + "/index.pkl", "rb"),
+            }
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
+            )
+        else:
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), data=file_data
+            )
+        response.raise_for_status()
+    except requests.RequestException as e:
+        logging.error(f"Error uploading index: {e}")
+        raise
+    finally:
+        if settings.VECTOR_STORE == "faiss":
+            for file in files.values():
+                file.close()
 
 # Define the main function for ingesting and processing documents.
-def ingest_worker(self, directory, formats, name_job, filename, user):
+def ingest_worker(
+    self, directory, formats, name_job, filename, user, retriever="classic"
+):
     """
     Ingest and process documents.
 
@@ -69,46 +116,29 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         name_job (str): Name of the job for this ingestion task.
         filename (str): Name of the file to be ingested.
         user (str): Identifier for the user initiating the ingestion.
+        retriever (str): Type of retriever to use for processing the documents.
 
     Returns:
         dict: Information about the completed ingestion task, including input parameters and a "limited" flag.
     """
-    # directory = 'inputs' or 'temp'
-    # formats = [".rst", ".md"]
     input_files = None
     recursive = True
     limit = None
     exclude = True
-    # name_job = 'job1'
-    # filename = 'install.rst'
-    # user = 'local'
     sample = False
-    token_check = True
-    min_tokens = 150
-    max_tokens = 1250
-    recursion_depth = 2
     full_path = os.path.join(directory, user, name_job)
-    import sys
 
-    print(full_path, file=sys.stderr)
-    # check if API_URL env variable is set
+    logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job})
     file_data = {"name": name_job, "file": filename, "user": user}
-    response = requests.get(
-        urljoin(settings.API_URL, "/api/download"), params=file_data
-    )
-    # check if file is in the response
-    print(response, file=sys.stderr)
-    file = response.content
 
     if not os.path.exists(full_path):
         os.makedirs(full_path)
-    with open(os.path.join(full_path, filename), "wb") as f:
-        f.write(file)
+    download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename))
 
     # check if file is .zip and extract it
     if filename.endswith(".zip"):
         extract_zip_recursive(
-            os.path.join(full_path, filename), full_path, 0, recursion_depth
+            os.path.join(full_path, filename), full_path, 0, RECURSION_DEPTH
         )
 
     self.update_state(state="PROGRESS", meta={"current": 1})
@@ -122,41 +152,33 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         exclude_hidden=exclude,
         file_metadata=metadata_from_filename,
     ).load_data()
-    raw_docs = group_split(
-        documents=raw_docs,
-        min_tokens=min_tokens,
-        max_tokens=max_tokens,
-        token_check=token_check,
+
+    chunker = Chunker(
+        chunking_strategy="classic_chunk",
+        max_tokens=MAX_TOKENS,
+        min_tokens=MIN_TOKENS,
+        duplicate_headers=False
     )
+    raw_docs = chunker.chunk(documents=raw_docs)
 
     docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+    id = ObjectId()
 
-    call_openai_api(docs, full_path, self)
+    embed_and_store_documents(docs, full_path, id, self)
     tokens = count_tokens_docs(docs)
     self.update_state(state="PROGRESS", meta={"current": 100})
 
     if sample:
         for i in range(min(5, len(raw_docs))):
-            print(raw_docs[i].text)
+            logging.info(f"Sample document {i}: {raw_docs[i]}")
 
-    # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
-    # and send them to the server (provide user and name in form)
-    file_data = {"name": name_job, "user": user, "tokens":tokens}
-    if settings.VECTOR_STORE == "faiss":
-        files = {
-            "file_faiss": open(full_path + "/index.faiss", "rb"),
-            "file_pkl": open(full_path + "/index.pkl", "rb"),
-        }
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
-        )
-        response = requests.get(
-            urljoin(settings.API_URL, "/api/delete_old?path=" + full_path)
-        )
-    else:
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), data=file_data
-        )
+    file_data.update({
+        "tokens": tokens,
+        "retriever": retriever,
+        "id": str(id),
+        "type": "local",
+    })
+    upload_index(full_path, file_data)
 
     # delete local
     shutil.rmtree(full_path)
@@ -170,68 +192,123 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
         "limited": False,
     }
 
-
-def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
-    token_check = True
-    min_tokens = 150
-    max_tokens = 1250
-    full_path = directory + "/" + user + "/" + name_job
-
+def remote_worker(
+    self,
+    source_data,
+    name_job,
+    user,
+    loader,
+    directory="temp",
+    retriever="classic",
+    sync_frequency="never",
+    operation_mode="upload",
+    doc_id=None,
+):  
+    full_path = os.path.join(directory, user, name_job)
     if not os.path.exists(full_path):
         os.makedirs(full_path)
+
     self.update_state(state="PROGRESS", meta={"current": 1})
+    try:
+        logging.info("Initializing remote loader with type: %s", loader)
+        remote_loader = RemoteCreator.create_loader(loader)
+        raw_docs = remote_loader.load_data(source_data)
 
-    remote_loader = RemoteCreator.create_loader(loader)
-    raw_docs = remote_loader.load_data(source_data)
-
-    docs = group_split(
-        documents=raw_docs,
-        min_tokens=min_tokens,
-        max_tokens=max_tokens,
-        token_check=token_check,
-    )
-    # docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
-    call_openai_api(docs, full_path, self)
-    tokens = count_tokens_docs(docs)
-    self.update_state(state="PROGRESS", meta={"current": 100})
-
-    # Proceed with uploading and cleaning as in the original function
-    file_data = {"name": name_job, "user": user, "tokens":tokens}
-    if settings.VECTOR_STORE == "faiss":
-        files = {
-            "file_faiss": open(full_path + "/index.faiss", "rb"),
-            "file_pkl": open(full_path + "/index.pkl", "rb"),
-        }
-        
-        requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
+        chunker = Chunker(
+            chunking_strategy="classic_chunk",
+            max_tokens=MAX_TOKENS,
+            min_tokens=MIN_TOKENS,
+            duplicate_headers=False
         )
-        requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path))
-    else:
-        requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
+        docs = chunker.chunk(documents=raw_docs)
+        docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+        tokens = count_tokens_docs(docs)
+        logging.info("Total tokens calculated: %d", tokens)
 
-    shutil.rmtree(full_path)
+        if operation_mode == "upload":
+            id = ObjectId()
+            embed_and_store_documents(docs, full_path, id, self)
+        elif operation_mode == "sync":
+            if not doc_id or not ObjectId.is_valid(doc_id):
+                logging.error("Invalid doc_id provided for sync operation: %s", doc_id)
+                raise ValueError("doc_id must be provided for sync operation.")
+            id = ObjectId(doc_id)
+            embed_and_store_documents(docs, full_path, id, self)
 
+        self.update_state(state="PROGRESS", meta={"current": 100})
+
+        file_data = {
+            "name": name_job,
+            "user": user,
+            "tokens": tokens,
+            "retriever": retriever,
+            "id": str(id),
+            "type": loader,
+            "remote_data": source_data,
+            "sync_frequency": sync_frequency,
+        }
+        upload_index(full_path, file_data)
+
+    except Exception as e:
+        logging.error("Error in remote_worker task: %s", str(e), exc_info=True)
+        raise
+
+    finally:
+        if os.path.exists(full_path):
+            shutil.rmtree(full_path)
+
+    logging.info("remote_worker task completed successfully")
     return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}
 
+def sync(
+    self,
+    source_data,
+    name_job,
+    user,
+    loader,
+    sync_frequency,
+    retriever,
+    doc_id=None,
+    directory="temp",
+):
+    try:
+        remote_worker(
+            self,
+            source_data,
+            name_job,
+            user,
+            loader,
+            directory,
+            retriever,
+            sync_frequency,
+            "sync",
+            doc_id,
+        )
+    except Exception as e:
+        logging.error(f"Error during sync: {e}")
+        return {"status": "error", "error": str(e)}
+    return {"status": "success"}
 
-def count_tokens_docs(docs):
-    # Here we convert the docs list to a string and calculate the number of tokens the string represents.
-    # docs_content = (" ".join(docs))
-    docs_content = ""
-    for doc in docs:
-        docs_content += doc.page_content
+def sync_worker(self, frequency):
+    sync_counts = Counter()
+    sources = sources_collection.find()
+    for doc in sources:
+        if doc.get("sync_frequency") == frequency:
+            name = doc.get("name")
+            user = doc.get("user")
+            source_type = doc.get("type")
+            source_data = doc.get("remote_data")
+            retriever = doc.get("retriever")
+            doc_id = str(doc.get("_id"))
+            resp = sync(
+                self, source_data, name, user, source_type, frequency, retriever, doc_id
+            )
+            sync_counts["total_sync_count"] += 1
+            sync_counts[
+                "sync_success" if resp["status"] == "success" else "sync_failure"
+            ] += 1
 
-    tokens, total_price = num_tokens_from_string(
-        string=docs_content, encoding_name="cl100k_base"
-    )
-    # Here we print the number of tokens and the approx user cost with some visually appealing formatting.
-    return tokens
-
-
-def num_tokens_from_string(string: str, encoding_name: str) -> int:
-    # Function to convert string to tokens and estimate user cost.
-    encoding = tiktoken.get_encoding(encoding_name)
-    num_tokens = len(encoding.encode(string))
-    total_price = (num_tokens / 1000) * 0.0004
-    return num_tokens, total_price
+    return {
+        key: sync_counts[key]
+        for key in ["total_sync_count", "sync_success", "sync_failure"]
+    }

docker-compose-azure.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docker-compose-dev.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
 
   redis:

docker-compose-local.yaml

@@ -1,8 +1,8 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend
+    volumes:
+    - ./frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING

docker-compose-mock.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend

docker-compose.yaml

@@ -1,8 +1,8 @@
-version: "3.9"
-
 services:
   frontend:
     build: ./frontend
+    volumes:
+      - ./frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING
@@ -20,6 +20,7 @@ services:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - CACHE_REDIS_URL=redis://redis:6379/2
     ports:
       - "7091:7091"
     volumes:
@@ -32,7 +33,7 @@ services:
 
   worker:
     build: ./application
-    command: celery -A application.app.celery worker -l INFO
+    command: celery -A application.app.celery worker -l INFO -B
     environment:
       - API_KEY=$API_KEY
       - EMBEDDINGS_KEY=$API_KEY
@@ -41,6 +42,7 @@ services:
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
+      - CACHE_REDIS_URL=redis://redis:6379/2
     depends_on:
       - redis
       - mongo

docs/README.md

@@ -46,6 +46,6 @@ yarn install
 yarn dev
 ```
 
-- Now, you should be able to view the docs on your local environment by visiting `http://localhost:5000`. You can explore the different markdown files and make changes as you see fit.
+- Now, you should be able to view the docs on your local environment by visiting `http://localhost:3000`. You can explore the different markdown files and make changes as you see fit.
 
 - **Footnotes:** This guide assumes you have Node.js and npm installed. The guide involves running a local server using yarn, and viewing the documentation offline. If you encounter any issues, it may be worth verifying your Node.js and npm installations and whether you have installed yarn correctly.

docs/package.json

@@ -7,8 +7,8 @@
   "license": "MIT",
   "dependencies": {
     "@vercel/analytics": "^1.1.1",
-    "docsgpt": "^0.3.7",
-    "next": "^14.1.1",
+    "docsgpt-react": "^0.4.9",
+    "next": "^14.2.22",
     "nextra": "^2.13.2",
     "nextra-theme-docs": "^2.13.2",
     "react": "^18.2.0",

docs/pages/Deploying/Development-Environment.md

@@ -0,0 +1,78 @@
+## Development Environments
+
+### Spin up Mongo and Redis
+
+For development, only two containers are used from [docker-compose.yaml](https://github.com/arc53/DocsGPT/blob/main/docker-compose.yaml) (by deleting all services except for Redis and Mongo).
+See file [docker-compose-dev.yaml](https://github.com/arc53/DocsGPT/blob/main/docker-compose-dev.yaml).
+
+Run
+
+```
+docker compose -f docker-compose-dev.yaml build
+docker compose -f docker-compose-dev.yaml up -d
+```
+
+### Run the Backend
+
+> [!Note]
+> Make sure you have Python 3.12 installed.
+
+1. Export required environment variables or prepare a `.env` file in the project folder:
+   - Copy [.env-template](https://github.com/arc53/DocsGPT/blob/main/application/.env-template) and create `.env`.
+
+(check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.)
+
+2. (optional) Create a Python virtual environment:
+   You can follow the [Python official documentation](https://docs.python.org/3/tutorial/venv.html) for virtual environments.
+
+a) On Mac OS and Linux
+
+```commandline
+python -m venv venv
+. venv/bin/activate
+```
+
+b) On Windows
+
+```commandline
+python -m venv venv
+ venv/Scripts/activate
+```
+
+3. Download embedding model and save it in the `model/` folder:
+You can use the script below, or download it manually from [here](https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip), unzip it and save it in the `model/` folder.
+
+```commandline
+wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip
+unzip mpnet-base-v2.zip -d model
+rm mpnet-base-v2.zip
+```
+
+4. Install dependencies for the backend:
+
+```commandline
+pip install -r application/requirements.txt
+```
+
+5. Run the app using `flask --app application/app.py run --host=0.0.0.0 --port=7091`.
+6. Start worker with `celery -A application.app.celery worker -l INFO`.
+
+> [!Note]
+> You can also launch the in a debugger mode in vscode by accessing SHIFT + CMD + D or SHIFT + Windows + D on windows and selecting Flask or Celery.
+
+
+### Start Frontend
+
+> [!Note]
+> Make sure you have Node version 16 or higher.
+
+1. Navigate to the [/frontend](https://github.com/arc53/DocsGPT/tree/main/frontend) folder.
+2. Install the required packages `husky` and `vite` (ignore if already installed).
+
+```commandline
+npm install husky -g
+npm install vite -g
+```
+
+3. Install dependencies by running `npm install --include=dev`.
+4. Run the app using `npm run dev`.