chore(deps): bump actions/setup-python from 5 to 6

Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6.
- [Release notes](https://github.com/actions/setup-python/releases)
- [Commits](https://github.com/actions/setup-python/compare/v5...v6)

---
updated-dependencies:
- dependency-name: actions/setup-python
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

.devcontainer/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.12-bookworm
+
+# Install Node.js 20.x
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install global npm packages
+RUN npm install -g husky vite
+
+# Create and activate Python virtual environment
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+WORKDIR /workspace

.devcontainer/devc-welcome.md

@@ -0,0 +1,49 @@
+# Welcome to DocsGPT Devcontainer
+
+Welcome to the DocsGPT development environment! This guide will help you get started quickly.
+
+## Starting Services
+
+To run DocsGPT, you need to start three main services: Flask (backend), Celery (task queue), and Vite (frontend). Here are the commands to start each service within the devcontainer:
+
+### Vite (Frontend)
+
+```bash
+cd frontend
+npm run dev -- --host
+```
+
+### Flask (Backend)
+
+```bash
+flask --app application/app.py run --host=0.0.0.0 --port=7091
+```
+
+### Celery (Task Queue)
+
+```bash
+celery -A application.app.celery worker -l INFO
+```
+
+## Github Codespaces Instructions
+
+### 1. Make Ports Public:
+
+Go to the "Ports" panel in Codespaces (usually located at the bottom of the VS Code window).
+
+For both port 5173 and 7091, right-click on the port and select "Make Public".
+
+![CleanShot 2025-02-12 at 09 46 14@2x](https://github.com/user-attachments/assets/00a34b16-a7ef-47af-9648-87a7e3008475)
+
+
+ ### 2. Update VITE_API_HOST:
+
+After making port 7091 public, copy the public URL provided by Codespaces for port 7091.
+
+Open the file frontend/.env.development.
+
+Find the line VITE_API_HOST=http://localhost:7091.
+
+Replace http://localhost:7091 with the public URL you copied from Codespaces.
+
+![CleanShot 2025-02-12 at 09 46 56@2x](https://github.com/user-attachments/assets/c472242f-1079-4cd8-bc0b-2d78db22b94c)

.devcontainer/devcontainer.json

@@ -0,0 +1,24 @@
+{
+	"name": "DocsGPT Dev Container",
+	"dockerComposeFile": ["docker-compose-dev.yaml", "docker-compose.override.yaml"],
+	"service": "dev",
+	"workspaceFolder": "/workspace",
+	"postCreateCommand": ".devcontainer/post-create-command.sh",
+	"forwardPorts": [7091, 5173, 6379, 27017],
+	"customizations": {
+	  "vscode": {
+		"extensions": [
+		  "ms-python.python",
+		  "ms-toolsai.jupyter",
+		  "esbenp.prettier-vscode",
+		  "dbaeumer.vscode-eslint"
+		]
+	  },
+	  "codespaces": {
+			"openFiles": [
+			".devcontainer/devc-welcome.md",
+			"CONTRIBUTING.md"
+			]
+		}
+	}
+  }

.devcontainer/docker-compose-dev.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
 
   redis:

.devcontainer/docker-compose.override.yaml

@@ -0,0 +1,40 @@
+version: '3.8'
+
+services:
+  dev:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ../:/workspace:cached
+    command: sleep infinity
+    depends_on:
+      redis:
+        condition: service_healthy
+      mongo:
+        condition: service_healthy
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/1
+      - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - CACHE_REDIS_URL=redis://redis:6379/2
+    networks:
+      - default
+
+  redis:
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 30s
+      retries: 5
+
+  mongo:
+    healthcheck:
+      test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
+      interval: 5s
+      timeout: 30s
+      retries: 5
+
+networks:
+  default:
+    name: docsgpt-dev-network

.devcontainer/post-create-command.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e  # Exit immediately if a command exits with a non-zero status
+
+if [ ! -f frontend/.env.development ]; then
+  cp -n .env-template frontend/.env.development || true # Assuming .env-template is in the root
+fi
+
+# Determine VITE_API_HOST based on environment
+if [ -n "$CODESPACES" ]; then
+  # Running in Codespaces
+  CODESPACE_NAME=$(echo "$CODESPACES" | cut -d'-' -f1) # Extract codespace name
+  PUBLIC_API_HOST="https://${CODESPACE_NAME}-7091.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN}"
+  echo "Setting VITE_API_HOST for Codespaces: $PUBLIC_API_HOST in frontend/.env.development"
+  sed -i "s|VITE_API_HOST=.*|VITE_API_HOST=$PUBLIC_API_HOST|" frontend/.env.development
+else
+  # Not running in Codespaces (local devcontainer)
+  DEFAULT_API_HOST="http://localhost:7091"
+  echo "Setting VITE_API_HOST for local dev: $DEFAULT_API_HOST in frontend/.env.development"
+  sed -i "s|VITE_API_HOST=.*|VITE_API_HOST=$DEFAULT_API_HOST|" frontend/.env.development
+fi
+
+
+mkdir -p model
+if [ ! -d model/all-mpnet-base-v2 ]; then
+    wget -q https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip -O model/mpnet-base-v2.zip
+    unzip -q model/mpnet-base-v2.zip -d model
+    rm model/mpnet-base-v2.zip
+fi
+pip install -r application/requirements.txt
+cd frontend
+npm install --include=dev

.env-template

@@ -1,8 +1,8 @@
-OPENAI_API_KEY=<LLM api key (for example, open ai key)>
-SELF_HOSTED_MODEL=false
+API_KEY=<LLM api key (for example, open ai key)>
+LLM_NAME=docsgpt
 VITE_API_STREAMING=true
 
-#For Azure
+#For Azure (you can delete it if you don't use Azure)
 OPENAI_API_BASE=
 OPENAI_API_VERSION=
 AZURE_DEPLOYMENT_NAME=

.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: arc53

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,138 @@
+name: "🐛 Bug Report"
+description: "Submit a bug report to help us improve"
+title: "🐛 Bug Report: "
+labels: ["type: bug"]
+body:
+  - type: markdown
+    attributes:
+      value: We value your time and your efforts to submit this bug report is appreciated. 🙏
+
+  - type: textarea
+    id: description
+    validations:
+      required: true
+    attributes:
+      label: "📜 Description"
+      description: "A clear and concise description of what the bug is."
+      placeholder: "It bugs out when ..."
+
+  - type: textarea
+    id: steps-to-reproduce
+    validations:
+      required: true
+    attributes:
+      label: "👟 Reproduction steps"
+      description: "How do you trigger this bug? Please walk us through it step by step."
+      placeholder: "1. Go to '...'
+        2. Click on '....'
+        3. Scroll down to '....'
+        4. See error"
+
+  - type: textarea
+    id: expected-behavior
+    validations:
+      required: true
+    attributes:
+      label: "👍 Expected behavior"
+      description: "What did you think should happen?"
+      placeholder: "It should ..."
+
+  - type: textarea
+    id: actual-behavior
+    validations:
+      required: true
+    attributes:
+      label: "👎 Actual Behavior with Screenshots"
+      description: "What did actually happen? Add screenshots, if applicable."
+      placeholder: "It actually ..."
+
+  - type: dropdown
+    id: operating-system
+    attributes:
+      label: "💻 Operating system"
+      description: "What OS is your app running on?"
+      options:
+        - Linux
+        - MacOS
+        - Windows
+        - Something else
+    validations:
+      required: true
+
+  - type: dropdown
+    id: browsers
+    attributes:
+      label: What browsers are you seeing the problem on?
+      multiple: true
+      options:
+        - Firefox
+        - Chrome
+        - Safari
+        - Microsoft Edge
+        - Something else
+
+  - type: dropdown
+    id: dev-environment
+    validations:
+      required: true
+    attributes:
+      label: "🤖 What development environment are you experiencing this bug on?"
+      options:
+        - Docker
+        - Local dev server
+
+  - type: textarea
+    id: env-vars
+    validations:
+      required: false
+    attributes:
+      label: "🔒 Did you set the correct environment variables in the right path? List the environment variable names (not values please!)"
+      description: "Please refer to the [Project setup instructions](https://github.com/arc53/DocsGPT#quickstart) if you are unsure."
+      placeholder: "It actually ..."
+
+  - type: textarea
+    id: additional-context
+    validations:
+      required: false
+    attributes:
+      label: "📃 Provide any additional context for the Bug."
+      description: "Add any other context about the problem here."
+      placeholder: "It actually ..."
+
+  - type: textarea
+    id: logs
+    validations:
+      required: false
+    attributes:
+      label: 📖 Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+
+  - type: checkboxes
+    id: no-duplicate-issues
+    attributes:
+      label: "👀 Have you spent some time to check if this bug has been raised before?"
+      options:
+        - label: "I checked and didn't find similar issue"
+          required: true
+
+  - type: dropdown
+    id: willing-to-submit-pr
+    attributes:
+      label: 🔗 Are you willing to submit PR?
+      description: This is absolutely not required, but we are happy to guide you in the contribution process.
+      options: # Added options key
+        - "Yes, I am willing to submit a PR!"
+        - "No"
+    validations:
+      required: false
+
+
+  - type: checkboxes
+    id: terms
+    attributes:
+      label: 🧑‍⚖️ Code of Conduct
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/arc53/DocsGPT/blob/main/CODE_OF_CONDUCT.md)
+      options:
+        - label: I agree to follow this project's Code of Conduct
+          required: true

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,54 @@
+name: 🚀 Feature
+description: "Submit a proposal for a new feature"
+title: "🚀 Feature: "
+labels: [feature]
+body:
+  - type: markdown
+    attributes:
+      value: We value your time and your efforts to submit this bug report is appreciated. 🙏
+  - type: textarea
+    id: feature-description
+    validations:
+      required: true
+    attributes:
+      label: "🔖 Feature description"
+      description: "A clear and concise description of what the feature is."
+      placeholder: "You should add ..."
+  - type: textarea
+    id: pitch
+    validations:
+      required: true
+    attributes:
+      label: "🎤 Why is this feature needed ?"
+      description: "Please explain why this feature should be implemented and how it would be used. Add examples, if applicable."
+      placeholder: "In my use-case, ..."
+  - type: textarea
+    id: solution
+    validations:
+      required: true
+    attributes:
+      label: "✌️ How do you aim to achieve this?"
+      description: "A clear and concise description of what you want to happen."
+      placeholder: "I want this feature to, ..."
+  - type: textarea
+    id: alternative
+    validations:
+      required: false
+    attributes:
+      label: "🔄️ Additional Information"
+      description: "A clear and concise description of any alternative solutions or additional solutions you've considered."
+      placeholder: "I tried, ..."
+  - type: checkboxes
+    id: no-duplicate-issues
+    attributes:
+      label: "👀 Have you spent some time to check if this feature request has been raised before?"
+      options:
+        - label: "I checked and didn't find similar issue"
+          required: true
+  - type: dropdown
+    id: willing-to-submit-pr
+    attributes:
+      label: Are you willing to submit PR?
+      description: This is absolutely not required, but we are happy to guide you in the contribution process.
+      options:
+        - "Yes I am willing to submit a PR!"

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,5 @@
+- **What kind of change does this PR introduce?** (Bug fix, feature, docs update, ...)
+
+- **Why was this change needed?** (You can also link to an open issue here)
+
+- **Other information**:

.github/dependabot.yml

@@ -0,0 +1,19 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/application" # Location of package manifests
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm" # See documentation for possible values
+    directory: "/frontend" # Location of package manifests
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/holopin.yml

@@ -0,0 +1,11 @@
+organization: docsgpt
+defaultSticker: cm1ulwkkl180570cl82rtzympu
+stickers:
+  - id: cm1ulwkkl180570cl82rtzympu
+    alias: contributor2024
+  - id: cm1ureg8o130450cl8c1po6mil
+    alias: api
+  - id: cm1urhmag148240cl8yvqxkthx
+    alias: lpc
+  - id: cm1urlcpq622090cl2tvu4w71y
+    alias: lexeu

.github/labeler.yml

@@ -0,0 +1,31 @@
+repo:
+- changed-files:
+  - any-glob-to-any-file: '*'
+
+github:
+- changed-files:
+  - any-glob-to-any-file: '.github/**/*'
+
+application:
+- changed-files:
+  - any-glob-to-any-file: 'application/**/*'
+
+docs:
+- changed-files:
+  - any-glob-to-any-file: 'docs/**/*'
+
+extensions:
+- changed-files:
+  - any-glob-to-any-file: 'extensions/**/*'
+
+frontend:
+- changed-files:
+  - any-glob-to-any-file: 'frontend/**/*'
+
+scripts:
+- changed-files:
+  - any-glob-to-any-file: 'scripts/**/*'
+
+tests:
+- changed-files:
+  - any-glob-to-any-file: 'tests/**/*'

.github/workflows/bandit.yaml

@@ -0,0 +1,40 @@
+name: Bandit Security Scan
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  bandit_scan:
+    if: ${{ github.repository == 'arc53/DocsGPT' }}
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      actions: read
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install bandit  # Bandit is needed for this action
+          if [ -f application/requirements.txt ]; then pip install -r application/requirements.txt; fi
+
+      - name: Run Bandit scan
+        uses: PyCQA/bandit-action@v1
+        with:
+          severity: medium
+          confidence: medium
+          targets: application/
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -1,49 +1,112 @@
 name: Build and push DocsGPT Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
-  deploy:
+  build:
     if: github.repository == 'arc53/DocsGPT'
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            suffix: amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            suffix: arm64
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
-
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+      - name: Set up QEMU  # Only needed for emulation, not for native arm64 builds
+        if: matrix.platform == 'linux/arm64'
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Runs a single command using the runners shell
-      - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+      - name: Build and push platform-specific images
+        uses: docker/build-push-action@v6
         with:
           file: './application/Dockerfile'
-          platforms: linux/amd64
+          platforms: ${{ matrix.platform }}
           context: ./application
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
+            ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
+          provenance: false
+          sbom: false
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+          cache-to: type=inline
+
+  manifest:
+    if: github.repository == 'arc53/DocsGPT'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest for DockerHub
+        run: |
+          set -e
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }} \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+
+      - name: Create and push manifest for ghcr.io
+        run: |
+          set -e
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }} \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:latest \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:latest

.github/workflows/cife.yml

@@ -1,48 +1,112 @@
 name: Build and push DocsGPT-FE Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
-  deploy:
-    runs-on: ubuntu-latest
+  build:
+    if: github.repository == 'arc53/DocsGPT'
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            suffix: amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            suffix: arm64
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
-
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+      - name: Set up QEMU # Only needed for emulation, not for native arm64 builds
+        if: matrix.platform == 'linux/arm64'
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Runs a single command using the runners shell
-      - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+      - name: Build and push platform-specific images
+        uses: docker/build-push-action@v6
         with:
           file: './frontend/Dockerfile'
-          platforms: linux/amd64
+          platforms: ${{ matrix.platform }}
           context: ./frontend
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
+          provenance: false
+          sbom: false
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+          cache-to: type=inline
+
+  manifest:
+    if: github.repository == 'arc53/DocsGPT'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest for DockerHub
+        run: |
+          set -e
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }} \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+
+      - name: Create and push manifest for ghcr.io
+        run: |
+          set -e
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }} \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest

.github/workflows/docker-develop-build.yml

@@ -0,0 +1,100 @@
+name: Build and push multi-arch DocsGPT Docker image
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    if: github.repository == 'arc53/DocsGPT'
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            suffix: amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            suffix: arm64
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push platform-specific images
+        uses: docker/build-push-action@v6
+        with:
+          file: './application/Dockerfile'
+          platforms: ${{ matrix.platform }}
+          context: ./application
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-${{ matrix.suffix }}
+            ghcr.io/${{ github.repository_owner }}/docsgpt:develop-${{ matrix.suffix }}
+          provenance: false
+          sbom: false
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+          cache-to: type=inline
+
+  manifest:
+    if: github.repository == 'arc53/DocsGPT'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+        
+      - name: Create and push manifest for DockerHub
+        run: |
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+
+      - name: Create and push manifest for ghcr.io
+        run: |
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:develop \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:develop

.github/workflows/docker-develop-fe-build.yml

@@ -0,0 +1,104 @@
+name: Build and push DocsGPT FE Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    if: github.repository == 'arc53/DocsGPT'
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            suffix: amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            suffix: arm64
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU # Only needed for emulation, not for native arm64 builds
+        if: matrix.platform == 'linux/arm64'
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push platform-specific images
+        uses: docker/build-push-action@v6
+        with:
+          file: './frontend/Dockerfile'
+          platforms: ${{ matrix.platform }}
+          context: ./frontend
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop-${{ matrix.suffix }}
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop-${{ matrix.suffix }}
+          provenance: false
+          sbom: false
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+          cache-to: type=inline
+
+  manifest:
+    if: github.repository == 'arc53/DocsGPT'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest for DockerHub
+        run: |
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+
+      - name: Create and push manifest for ghcr.io
+        run: |
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop

.github/workflows/labeler.yml

@@ -0,0 +1,16 @@
+# https://github.com/actions/labeler
+name: Pull Request Labeler
+on:
+  - pull_request_target
+jobs:
+  triage:
+    if: github.repository == 'arc53/DocsGPT'
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/labeler@v5
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          sync-labels: true

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Lint with Ruff
         uses: chartboost/ruff-action@v1

.github/workflows/pytest.yml

@@ -6,11 +6,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.12"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -21,10 +21,10 @@ jobs:
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
       - name: Test with pytest and generate coverage report
         run: |
-          python -m pytest --cov=application --cov=scripts --cov=extensions --cov-report=xml
+          python -m pytest --cov=application --cov-report=xml
       - name: Upload coverage reports to Codecov
-        if: github.event_name == 'pull_request' && matrix.python-version == '3.11'
-        uses: codecov/codecov-action@v3
+        if: github.event_name == 'pull_request' && matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/sync_fork.yaml

@@ -17,7 +17,7 @@ jobs:
     steps:
       # Step 1: run a standard checkout action
       - name: Checkout target repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Step 2: run the sync action
       - name: Sync upstream changes

.gitignore

@@ -75,6 +75,7 @@ target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
+**/*.ipynb
 
 # IPython
 profile_default/
@@ -112,6 +113,7 @@ venv.bak/
 # Spyder project settings
 .spyderproject
 .spyproject
+.jwt_secret_key
 
 # Rope project settings
 .ropeproject
@@ -171,4 +173,5 @@ application/vectors/
 
 node_modules/
 .vscode/settings.json
-models/
+/models/
+model/

.vscode/launch.json

@@ -0,0 +1,54 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Docker Debug Frontend",
+      "request": "launch",
+      "type": "chrome",
+      "preLaunchTask": "docker-compose: debug:frontend",
+      "url": "http://127.0.0.1:5173",
+      "webRoot": "${workspaceFolder}/frontend",
+      "skipFiles": [
+        "<node_internals>/**"
+      ]
+    },
+    {
+        "name": "Flask Debugger",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "flask",
+        "env": {
+            "FLASK_APP": "application/app.py",
+            "PYTHONPATH": "${workspaceFolder}",
+            "FLASK_ENV": "development",
+            "FLASK_DEBUG": "1",
+            "FLASK_RUN_PORT": "7091",
+            "FLASK_RUN_HOST": "0.0.0.0"
+
+        },
+        "args": [
+            "run",
+            "--no-debugger"
+        ],
+        "cwd": "${workspaceFolder}",
+    },
+    {
+      "name": "Celery Debugger",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}",
+      },
+      "args": [
+        "-A",
+        "application.app.celery",
+        "worker",
+        "-l",
+        "INFO",
+        "--pool=solo"
+      ],
+      "cwd": "${workspaceFolder}"
+    }
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,21 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "docker-compose",
+      "label": "docker-compose: debug:frontend",
+      "dockerCompose": {
+        "up": {
+          "detached": true,
+          "services": [
+            "frontend"
+          ],
+          "build": true
+        },
+        "files": [
+          "${workspaceFolder}/docker-compose.yaml"
+        ]
+      }
+    }
+  ]
+}

CODE_OF_CONDUCT.md

@@ -2,58 +2,58 @@
 
 ## Our Pledge
 
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
+We as members, contributors and leaders pledge to make participation in our
+community, a harassment-free experience for everyone, regardless of age, body
 size, visible or invisible disability, ethnicity, sex characteristics, gender
 identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, religion, or sexual identity
+nationality, personal appearance, race, religion or sexual identity
 and orientation.
 
 We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
+diverse, inclusive and a healthy community.
 
 ## Our Standards
 
-Examples of behavior that contributes to a positive environment for our
+Examples of behavior that contribute to a positive environment for our
 community include:
 
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the
-  overall community
+## Demonstrating empathy and kindness towards other people
+1. Being respectful and open to differing opinions, viewpoints, and experiences
+2. Giving and gracefully accepting constructive feedback
+3. Taking accountability and offering apologies to those who have been impacted by our errors,
+  while also gaining insights from the situation
+4. Focusing on what is best not just for us as individuals but for the
+  community as a whole
 
 Examples of unacceptable behavior include:
 
-* The use of sexualized language or imagery, and sexual attention or
+1. The use of sexualized language or imagery, and sexual attention or
   advances of any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email
+2. Trolling, insulting or derogatory comments, and personal or political attacks
+3. Public or private harassment
+4. Publishing other's private information, such as a physical or email
   address, without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
+5. Other conduct which could reasonably be considered inappropriate in a
   professional setting
 
 ## Enforcement Responsibilities
 
 Community leaders are responsible for clarifying and enforcing our standards of
 acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
+response to any behavior that they deem inappropriate, threatening, offensive
 or harmful.
 
 Community leaders have the right and responsibility to remove, edit, or reject
 comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
+not aligned to this Code of Conduct and will communicate reasons for moderation
 decisions when appropriate.
 
 ## Scope
 
-This Code of Conduct applies within all community spaces, and also applies when
+This Code of Conduct applies within all community spaces and also applies when
 an individual is officially representing the community in public spaces.
 Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
+posting via an official social media account or acting as an appointed
 representative at an online or offline event.
 
 ## Enforcement
@@ -63,29 +63,27 @@ reported to the community leaders responsible for enforcement at
 contact@arc53.com.
 All complaints will be reviewed and investigated promptly and fairly.
 
-All community leaders are obligated to respect the privacy and security of the
+All community leaders are obligated to be respectful towards the privacy and security of the
 reporter of any incident.
 
 ## Enforcement Guidelines
 
 Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
+the consequences for any action that they deem in violation of this Code of Conduct:
 
 ### 1. Correction
+* **Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community space.
 
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
+* **Consequence**: A private, written warning from community leaders, providing
 clarity around the nature of the violation and an explanation of why the
 behavior was inappropriate. A public apology may be requested.
 
 ### 2. Warning
-
-**Community Impact**: A violation through a single incident or series
+* **Community Impact**: A violation through a single incident or series
 of actions.
 
-**Consequence**: A warning with consequences for continued behavior. No
+* **Consequence**: A warning with consequences for continued behavior. No
 interaction with the people involved, including unsolicited interaction with
 those enforcing the Code of Conduct, for a specified period of time. This
 includes avoiding interactions in community spaces as well as external channels
@@ -93,23 +91,21 @@ like social media. Violating these terms may lead to a temporary or
 permanent ban.
 
 ### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
+* **Community Impact**: A serious violation of community standards, including
 sustained inappropriate behavior.
 
-**Consequence**: A temporary ban from any sort of interaction or public
+* **Consequence**: A temporary ban from any sort of interaction or public
 communication with the community for a specified period of time. No public or
 private interaction with the people involved, including unsolicited interaction
 with those enforcing the Code of Conduct, is allowed during this period.
 Violating these terms may lead to a permanent ban.
 
 ### 4. Permanent Ban
+* **Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,harassment of an
+individual or aggression towards or disparagement of classes of individuals.
 
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior,  harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within
+* **Consequence**: A permanent ban from any sort of public interaction within
 the community.
 
 ## Attribution

CONTRIBUTING.md

@@ -1,44 +1,151 @@
-# Welcome to DocsGPT Contributing guideline
+# Welcome to DocsGPT Contributing Guidelines
 
-Thank you for choosing this project to contribute to, we are all very grateful!
+Thank you for choosing to contribute to DocsGPT! We are all very grateful! 
 
 # We accept different types of contributions
 
-📣 Discussions - where you can start a new topic or answer some questions
+📣 **Discussions** - Engage in conversations, start new topics, or help answer questions.
 
-🐞 Issues - This is how we track tasks, sometimes it is bugs that need fixing, and sometimes it is new features
+🐞 **Issues** - This is where we keep track of tasks. It could be bugs, fixes or suggestions for new features.
 
-🛠️ Pull requests - This is how you can suggest changes to our repository, to work on existing issues or add new features
+🛠️ **Pull requests** - Suggest changes to our repository, either by working on existing issues or adding new features.
 
-📚 Wiki - where we have our documentation
+📚 **Wiki** - This is where our documentation resides.
 
 
 ## 🐞 Issues and Pull requests
 
-We value contributions to our issues in the form of discussion or suggestion, we recommend that you check out existing issues and our [Roadmap](https://github.com/orgs/arc53/projects/2)
+- We value contributions in the form of discussions or suggestions. We recommend taking a look at existing issues and our [roadmap](https://github.com/orgs/arc53/projects/2).
 
-If you want to contribute by writing code there are a few things that you should know before doing it:
-We have frontend (React, Vite) and Backend (python)
 
-### If you are looking to contribute to Frontend (⚛️React, Vite):
-The current frontend is being migrated from /application to /frontend with a new design, so please contribute to the new one. Check out this [Milestone](https://github.com/arc53/DocsGPT/milestone/1) and its issues also [Figma](https://www.figma.com/file/OXLtrl1EAy885to6S69554/DocsGPT?node-id=0%3A1&t=hjWVuxRg9yi5YkJ9-1)
-Please try to follow the guidelines.
+- If you're interested in contributing code, here are some important things to know:
 
-### If you are looking to contribute to Backend (🐍Python):
-* Check out our issues, and contribute to /application or /scripts (ignore old  ingest_rst.py ingest_rst_sphinx.py files, they will be deprecated soon)
-* All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [/tests](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
-* Before submitting your PR make sure that after you ingested some test data it is queryable.
+- We have a frontend built on React (Vite) and a backend in Python.
 
+  
+Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
+
+### 👨‍💻 If you're interested in contributing code, here are some important things to know:
+
+For instructions on setting up a development environment, please refer to our [Development Deployment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).
+
+Tech Stack Overview:
+
+- 🌐 Frontend: Built with React (Vite) ⚛️,
+
+- 🖥 Backend: Developed in Python 🐍
+
+### 🌐 Frontend Contributions (⚛️ React, Vite)
+
+*   The updated Figma design can be found [here](https://www.figma.com/file/OXLtrl1EAy885to6S69554/DocsGPT?node-id=0%3A1&t=hjWVuxRg9yi5YkJ9-1).  Please try to follow the guidelines.
+*   **Coding Style:** We follow a strict coding style enforced by ESLint and Prettier. Please ensure your code adheres to the configuration provided in our repository's `fronetend/.eslintrc.js` file.  We recommend configuring your editor with ESLint and Prettier to help with this.
+* **Component Structure:** Strive for small, reusable components.  Favor functional components and hooks over class components where possible.
+* **State Management** If you need to add stores, please use Redux.
+
+### 🖥 Backend Contributions (🐍 Python)
+
+- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) 
+- All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
+- Before submitting your Pull Request, ensure it can be queried after ingesting some test data.
+- **Coding Style:** We adhere to the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style guide for Python code. We use `ruff` as our linter and code formatter.  Please ensure your code is formatted correctly and passes `ruff` checks before submitting.
+- **Type Hinting:**  Please use type hints for all function arguments and return values. This improves code readability and helps catch errors early.  Example:
+
+    ```python
+    def my_function(name: str, count: int) -> list[str]:
+        ...
+    ```
+- **Docstrings:**  All functions and classes should have docstrings explaining their purpose, parameters, and return values.  We prefer the [Google style docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). Example:
+
+    ```python
+    def my_function(name: str, count: int) -> list[str]:
+        """Does something with a name and a count.
+
+        Args:
+            name: The name to use.
+            count: The number of times to do it.
+
+        Returns:
+            A list of strings.
+        """
+        ...
+    ```
+  
 ### Testing
-To run unit tests, from the root of the repository execute:
+
+To run unit tests from the root of the repository, execute:
 ```
 python -m pytest
 ```
 
-### Workflow:
-Create a fork, make changes on your forked repository, and submit changes in the form of a pull request.
+## Workflow 📈
+
+Here's a step-by-step guide on how to contribute to DocsGPT:
+
+1. **Fork the Repository:**
+   - Click the "Fork" button at the top-right of this repository to create your fork.
+
+2. **Clone the Forked Repository:**
+   - Clone the repository using:
+      ``` shell
+      git clone https://github.com/<your-github-username>/DocsGPT.git
+      ```
+
+3. **Keep your Fork in Sync:**
+   - Before you make any changes, make sure that your fork is in sync to avoid merge conflicts using:
+     ```shell
+     git remote add upstream https://github.com/arc53/DocsGPT.git
+     git pull upstream main
+     ```
+
+4. **Create and Switch to a New Branch:**
+   - Create a new branch for your contribution using:
+     ```shell
+     git checkout -b your-branch-name
+     ```
+
+5. **Make Changes:**
+   - Make the required changes in your branch.
+
+6. **Add Changes to the Staging Area:**
+   - Add your changes to the staging area using:
+     ```shell
+     git add .
+     ```
+
+7. **Commit Your Changes:**
+   - Commit your changes with a descriptive commit message using:
+     ```shell
+     git commit -m "Your descriptive commit message"
+     ```
+
+8. **Push Your Changes to the Remote Repository:**
+   - Push your branch with changes to your fork on GitHub using:
+     ```shell
+     git push origin your-branch-name
+     ```
+
+9. **Submit a Pull Request (PR):**
+   - Create a Pull Request from your branch to the main repository. Make sure to include a detailed description of your changes and reference any related issues.
+
+10. **Collaborate:**
+   - Be responsive to comments and feedback on your PR.
+   - Make necessary updates as suggested.
+   - Once your PR is approved, it will be merged into the main repository.
+
+11. **Testing:**
+   - Before submitting a Pull Request, ensure your code passes all unit tests.
+   - To run unit tests from the root of the repository, execute:
+     ```shell
+     python -m pytest
+     ```
+
+*Note: You should run the unit test only after making the changes to the backend code.*
+
+12. **Questions and Collaboration:**
+    - Feel free to join our Discord. We're very friendly and welcoming to new contributors, so don't hesitate to reach out.
+
+Thank you for considering contributing to DocsGPT! 🙏
 
 ## Questions/collaboration
-Please join our [Discord](https://discord.gg/n5BX8dh8rU) don't hesitate, we are very friendly and welcoming to new contributors.
-
-# Thank you so much for considering contributing to DocsGPT!🙏
+Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU). We're very friendly and welcoming to new contributors, so don't hesitate to reach out.
+# Thank you so much for considering to contributing DocsGPT!🙏

HACKTOBERFEST.md

@@ -1,31 +0,0 @@
-🎉 Join the Hacktoberfest with DocsGPT and Earn a Free T-shirt! 🎉
-
-Welcome, contributors! We're excited to announce that DocsGPT is participating in Hacktoberfest. Get involved by submitting a **meaningful** pull request, and earn a free shirt in return!
-📜 Here's How to Contribute:
-
-    🛠️ Code: This is the golden ticket! Make meaningful contributions through PRs.
-    📚 Wiki: Improve our documentation, Create a guide or change existing documentation.
-    🖥️ Design: Improve the UI/UX, or design a new feature.
-
-📝 Guidelines for Pull Requests:
-
-Familiarize yourself with the current contributions and our [Roadmap](https://github.com/orgs/arc53/projects/2).
-
-Deciding to contribute with code? Here are some insights based on the area of your interest:
-
-Frontend (⚛️React, Vite):
-    Most of the code is located in /frontend folder. You can also check out our React extension in /extensions/react-widget.
-    For design references, here's the [Figma](https://www.figma.com/file/OXLtrl1EAy885to6S69554/DocsGPT?node-id=0%3A1&t=hjWVuxRg9yi5YkJ9-1).
-    Ensure you adhere to the established guidelines.
-
-Backend (🐍Python):
-    Focus on /application or /scripts. However, avoid the files ingest_rst.py and ingest_rst_sphinx.py as they are soon to be deprecated.
-    Newly added code should come with relevant unit tests (pytest).
-    Refer to the /tests folder for test suites.
-
-Check out [Contributing Guidelines](https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md)
-
-
-Don't be shy! Hop into our [Discord](https://discord.gg/n5BX8dh8rU) Server. We're a friendly bunch and eager to assist newcomers.
-
-Big thanks for considering contributing to DocsGPT during Hacktoberfest! 🙏 Your effort can earn you a swanky new t-shirt. 🎁 Let's code together! 🚀

README.md

@@ -3,154 +3,159 @@
 </h1>
 
 <p align="center">
-  <strong>Open-Source Documentation Assistant</strong>
+  <strong>Private AI for agents, assistants and enterprise search</strong>
 </p>
 
 <p align="left">
-  <strong>DocsGPT</strong> is a cutting-edge open-source solution that streamlines the process of finding information in project documentation. With its integration of the powerful <strong>GPT</strong> models, developers can easily ask questions about a project and receive accurate answers.
-  
-Say goodbye to time-consuming manual searches, and let <strong>DocsGPT</strong> help you quickly find the information you need. Try it out and see how it revolutionizes your project documentation experience. Contribute to its development and be a part of the future of AI-powered assistance.
+  <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> is an open-source AI platform for building intelligent agents and assistants. Features Agent Builder, deep research tools, document analysis (PDF, Office, web content), Multi-model support (choose your provider or run locally), and rich API connectivity for agents with actionable tools and integrations. Deploy anywhere with complete privacy control.
 </p>
 
 <div align="center">
   
-  <a href="https://discord.gg/n5BX8dh8rU">![example1](https://img.shields.io/github/stars/arc53/docsgpt?style=social)</a>
-  <a href="https://discord.gg/n5BX8dh8rU">![example2](https://img.shields.io/github/forks/arc53/docsgpt?style=social)</a>
-  <a href="https://discord.gg/n5BX8dh8rU">![example3](https://img.shields.io/github/license/arc53/docsgpt)</a>
-  <a href="https://discord.gg/n5BX8dh8rU">![example3](https://img.shields.io/discord/1070046503302877216)</a>
+  <a href="https://github.com/arc53/DocsGPT">![link to main GitHub showing Stars number](https://img.shields.io/github/stars/arc53/docsgpt?style=social)</a>
+  <a href="https://github.com/arc53/DocsGPT">![link to main GitHub showing Forks number](https://img.shields.io/github/forks/arc53/docsgpt?style=social)</a>
+  <a href="https://github.com/arc53/DocsGPT/blob/main/LICENSE">![link to license file](https://img.shields.io/github/license/arc53/docsgpt)</a>
+  <a href="https://www.bestpractices.dev/projects/9907"><img src="https://www.bestpractices.dev/projects/9907/badge"></a>
+  <a href="https://discord.gg/n5BX8dh8rU">![link to discord](https://img.shields.io/discord/1070046503302877216)</a>
+  <a href="https://twitter.com/docsgptai">![X (formerly Twitter) URL](https://img.shields.io/twitter/follow/docsgptai)</a>
 
+<a href="https://docs.docsgpt.cloud/quickstart">⚡️ Quickstart</a> • <a href="https://app.docsgpt.cloud/">☁️ Cloud Version</a> • <a href="https://discord.gg/n5BX8dh8rU">💬 Discord</a>
+<br>
+<a href="https://docs.docsgpt.cloud/">📖 Documentation</a> • <a href="https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md">👫 Contribute</a> • <a href="https://blog.docsgpt.cloud/">🗞 Blog</a>
+<br>
 
-  
 </div>
-
-### Enterprise Solutions: 
-
-When deploying your DocsGPT to a live environment, we're eager to provide personalized assistance. Reach out to us via email [here]( mailto:contact@arc53.com?subject=DocsGPT%20Enterprise&body=Hi%20we%20are%20%3CCompany%20name%3E%20and%20we%20want%20to%20build%20%3CSolution%3E%20with%20DocsGPT) to discuss your project further, and our team will connect with you shortly.
-
-### [🎉 Join the Hacktoberfest with DocsGPT and Earn a Free T-shirt! 🎉](https://github.com/arc53/DocsGPT/blob/main/HACKTOBERFEST.md)
-
-![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
-
+<div align="center">
+<img src="https://d3dg1063dc54p9.cloudfront.net/videos/demov7.gif" alt="video-example-of-docs-gpt" width="800" height="450">
+</div>
+<h3 align="left">
+  <strong>Key Features:</strong>
+</h3>
+<ul align="left">
+    <li><strong>🗂️ Wide Format Support:</strong> Reads PDF, DOCX, CSV, XLSX, EPUB, MD, RST, HTML, MDX, JSON, PPTX, and images.</li>
+    <li><strong>🌐 Web & Data Integration:</strong> Ingests from URLs, sitemaps, Reddit, GitHub and web crawlers.</li>
+    <li><strong>✅ Reliable Answers:</strong> Get accurate, hallucination-free responses with source citations viewable in a clean UI.</li>
+    <li><strong>🔑 Streamlined API Keys:</strong>  Generate keys linked to your settings, documents, and models, simplifying chatbot and integration setup.</li>
+    <li><strong>🔗 Actionable Tooling:</strong> Connect to APIs, tools, and other services to enable LLM actions.</li>
+    <li><strong>🧩 Pre-built Integrations:</strong> Use readily available HTML/React chat widgets, search tools, Discord/Telegram bots, and more.</li>
+    <li><strong>🔌 Flexible Deployment:</strong> Works with major LLMs (OpenAI, Google, Anthropic) and local models (Ollama, llama_cpp).</li>
+    <li><strong>🏢 Secure & Scalable:</strong> Run privately and securely with Kubernetes support, designed for enterprise-grade reliability.</li>
+</ul>
 
 ## Roadmap
 
-You can find our [Roadmap](https://github.com/orgs/arc53/projects/2) here. Please don't hesitate to contribute or create issues, it helps us make DocsGPT better!
+- [x] Full GoogleAI compatibility (Jan 2025)
+- [x] Add tools (Jan 2025)
+- [x] Manually updating chunks in the app UI (Feb 2025)
+- [x] Devcontainer for easy development (Feb 2025)
+- [x] ReACT agent (March 2025)
+- [x] Chatbots menu re-design to handle tools, agent types, and more (April 2025)
+- [x] New input box in the conversation menu (April 2025)
+- [x] Add triggerable actions / tools (webhook) (April 2025)
+- [x] Agent optimisations (May 2025)
+- [x] Filesystem sources update (July 2025)
+- [x] Json Responses (August 2025)
+- [ ] Sharepoint integration (August 2025)
+- [ ] MCP support (August 2025)
+- [ ] Add OAuth 2.0 authentication for tools and sources (August 2025)
+- [ ] Agent scheduling
 
-## Our open source models optimised for DocsGPT:
+You can find our full roadmap [here](https://github.com/orgs/arc53/projects/2). Please don't hesitate to contribute or create issues, it helps us improve DocsGPT!
 
-| Name              | Base Model | Requirements (or similar)                        |
-|-------------------|------------|----------------------------------------------------------|
-| [Docsgpt-7b-falcon](https://huggingface.co/Arc53/docsgpt-7b-falcon)  | Falcon-7b  |  1xA10G gpu   |
-| [Docsgpt-14b](https://huggingface.co/Arc53/docsgpt-14b)              | llama-2-14b    | 2xA10 gpu's   |
-| [Docsgpt-40b-falcon](https://huggingface.co/Arc53/docsgpt-40b-falcon)       | falcon-40b     | 8xA10G gpu's  |
+### Production Support / Help for Companies:
 
+We're eager to provide personalized assistance when deploying your DocsGPT to a live environment.
 
-If you don't have enough resources to run it you can use bitsnbytes to quantize
+[Get a Demo :wave:](https://www.docsgpt.cloud/contact)⁠
 
+[Send Email :email:](mailto:support@docsgpt.cloud?subject=DocsGPT%20support%2Fsolutions)
 
-## Features
+## Join the Lighthouse Program 🌟
 
-![Group 9](https://user-images.githubusercontent.com/17906039/220427472-2644cff4-7666-46a5-819f-fc4a521f63c7.png)
+Calling all developers and GenAI innovators! The **DocsGPT Lighthouse Program** connects technical leaders actively deploying or extending DocsGPT in real-world scenarios. Collaborate directly with our team to shape the roadmap, access priority support, and build enterprise-ready solutions with exclusive community insights.
 
-
-## Useful links
- [Live preview](https://docsgpt.arc53.com/)
- 
- [Join Our Discord](https://discord.gg/n5BX8dh8rU)
- 
- [Guides](https://docs.docsgpt.co.uk/)
-
- [Interested in contributing?](https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md)
-
- [How to use any other documentation](https://docs.docsgpt.co.uk/Guides/How-to-train-on-other-documentation)
-
- [How to host it locally (so all data will stay on-premises)](https://docs.docsgpt.co.uk/Guides/How-to-use-different-LLM)
-
-
-## Project structure
-- Application - Flask app (main application)
-
-- Extensions - Chrome extension
-
-- Scripts - Script that creates similarity search index and store for other libraries. 
-
-- Frontend - Frontend uses Vite and React
+[Learn More & Apply →](https://docs.google.com/forms/d/1KAADiJinUJ8EMQyfTXUIGyFbqINNClNR3jBNWq7DgTE)
 
 ## QuickStart
 
-Note: Make sure you have Docker installed
+> [!Note]
+> Make sure you have [Docker](https://docs.docker.com/engine/install/) installed
 
-On Mac OS or Linux just write:
+A more detailed [Quickstart](https://docs.docsgpt.cloud/quickstart) is available in our documentation
 
-`./setup.sh`
+1. **Clone the repository:**
 
-It will install all the dependencies and give you an option to download local model or use OpenAI
-
-Otherwise refer to this Guide:
-
-1. Download and open this repository with `git clone https://github.com/arc53/DocsGPT.git`
-2. Create a .env file in your root directory and set the env variable OPENAI_API_KEY with your OpenAI API key and  VITE_API_STREAMING to true or false, depending on if you want streaming answers or not
-   It should look like this inside:
-   
+   ```bash
+   git clone https://github.com/arc53/DocsGPT.git
+   cd DocsGPT
    ```
-   OPENAI_API_KEY=Yourkey
-   VITE_API_STREAMING=true
-   SELF_HOSTED_MODEL=false
+
+**For macOS and Linux:**
+
+2. **Run the setup script:**
+
+   ```bash
+   ./setup.sh
    ```
-   See optional environment variables in the `/.env-template` and `/application/.env_sample` files.
-3. Run `./run-with-docker-compose.sh`
-4. Navigate to http://localhost:5173/
 
-To stop just run Ctrl + C
+**For Windows:**
 
-## Development environments
+2. **Run the PowerShell setup script:**
 
-### Spin up mongo and redis
-For development only 2 containers are used from docker-compose.yaml (by deleting all services except for Redis and Mongo). 
-See file [docker-compose-dev.yaml](./docker-compose-dev.yaml).
+   ```powershell
+   PowerShell -ExecutionPolicy Bypass -File .\setup.ps1
+   ```
 
-Run
-```
-docker compose -f docker-compose-dev.yaml build
-docker compose -f docker-compose-dev.yaml up -d
+Either script will guide you through setting up DocsGPT. Four options available: using the public API, running locally, connecting to a local inference engine, or using a cloud API provider. Scripts will automatically configure your `.env` file and handle necessary downloads and installations based on your chosen option.
+
+**Navigate to http://localhost:5173/**
+
+To stop DocsGPT, open a terminal in the `DocsGPT` directory and run:
+
+```bash
+docker compose -f deployment/docker-compose.yaml down
 ```
 
-### Run the backend
+(or use the specific `docker compose down` command shown after running the setup script).
 
-Make sure you have Python 3.10 or 3.11 installed.
+> [!Note]
+> For development environment setup instructions, please refer to the [Development Environment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).
 
-1. Export required environment variables
-```commandline
-export CELERY_BROKER_URL=redis://localhost:6379/0   
-export CELERY_RESULT_BACKEND=redis://localhost:6379/1
-export MONGO_URI=mongodb://localhost:27017/docsgpt
-export FLASK_APP=application/app.py
-export FLASK_DEBUG=true
-```
-2. Prepare .env file
-Copy `.env_sample` and create `.env` with your OpenAI API token
-3. (optional) Create a Python virtual environment
-```commandline
-python -m venv venv
-. venv/bin/activate
-```
-4. Change to `application/` subdir and install dependencies for the backend
-```commandline
-pip install -r application/requirements.txt
-```
-5. Run the app `flask run --host=0.0.0.0 --port=7091`
-6. Start worker with `celery -A application.app.celery worker -l INFO`
+## Contributing
 
-### Start frontend 
-Make sure you have Node version 16 or higher.
+Please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file for information about how to get involved. We welcome issues, questions, and pull requests.
 
-1. Navigate to `/frontend` folder
-2. Install dependencies
-`npm install`
-3. Run the app 
-`npm run dev`
+## Architecture
 
+![Architecture chart](https://github.com/user-attachments/assets/fc6a7841-ddfc-45e6-b5a0-d05fe648cbe2)
 
+## Project Structure
 
-Built with [🦜️🔗 LangChain](https://github.com/hwchase17/langchain)
+- Application - Flask app (main application).
 
+- Extensions - Extensions, like react widget or discord bot.
+
+- Frontend - Frontend uses <a href="https://vitejs.dev/">Vite</a> and <a href="https://react.dev/">React</a>.
+
+- Scripts - Miscellaneous scripts.
+
+## Code Of Conduct
+
+We as members, contributors, and leaders, pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. Please refer to the [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) file for more information about contributing.
+
+## Many Thanks To Our Contributors⚡
+
+<a href="https://github.com/arc53/DocsGPT/graphs/contributors" alt="View Contributors">
+  <img src="https://contrib.rocks/image?repo=arc53/DocsGPT" alt="Contributors" />
+</a>
+
+## License
+
+The source code license is [MIT](https://opensource.org/license/mit/), as described in the [LICENSE](LICENSE) file.
+
+<p>This project is supported by:</p>
+<p>
+  <a href="https://www.digitalocean.com/?utm_medium=opensource&utm_source=DocsGPT">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px">
+  </a>
+</p>

SECURITY.md

@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+Supported Versions:
+
+Currently, we support security patches by committing changes and bumping the version published on Github.
+
+## Reporting a Vulnerability
+
+Found a vulnerability? Please email us:
+
+security@arc53.com
+

application/.env_sample

@@ -1,9 +1,8 @@
 API_KEY=your_api_key
 EMBEDDINGS_KEY=your_api_key
-CELERY_BROKER_URL=redis://localhost:6379/0
-CELERY_RESULT_BACKEND=redis://localhost:6379/1
-MONGO_URI=mongodb://localhost:27017/docsgpt
 API_URL=http://localhost:7091
+FLASK_APP=application/app.py
+FLASK_DEBUG=true
 
 #For OPENAI on Azure
 OPENAI_API_BASE=

application/Dockerfile

@@ -1,23 +1,87 @@
-FROM python:3.10-slim-bullseye as builder
+# Builder Stage
+FROM ubuntu:24.04 as builder
 
-# Tiktoken requires Rust toolchain, so build it in a separate stage
-RUN apt-get update && apt-get install -y gcc curl
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && apt-get install --reinstall libc6-dev -y
-ENV PATH="/root/.cargo/bin:${PATH}"
-RUN pip install --upgrade pip && pip install tiktoken==0.3.3
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends gcc wget unzip libc6-dev python3.12 python3.12-venv && \
+    rm -rf /var/lib/apt/lists/* 
+
+# Verify Python installation and setup symlink
+RUN if [ -f /usr/bin/python3.12 ]; then \
+        ln -s /usr/bin/python3.12 /usr/bin/python; \
+    else \
+        echo "Python 3.12 not found"; exit 1; \
+    fi
+
+# Download and unzip the model
+RUN wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip && \
+    unzip mpnet-base-v2.zip -d models && \
+    rm mpnet-base-v2.zip
+
+# Install Rust
+RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y
+
+# Clean up to reduce container size
+RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements.txt
 COPY requirements.txt .
-RUN pip install -r requirements.txt
 
-FROM python:3.10-slim-bullseye
+# Setup Python virtual environment
+RUN python3.12 -m venv /venv
 
-# Copy pre-built packages and binaries from builder stage
-COPY --from=builder /usr/local/ /usr/local/
+# Activate virtual environment and install Python packages
+ENV PATH="/venv/bin:$PATH"
 
+# Install Python packages
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir tiktoken && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Final Stage
+FROM ubuntu:24.04 as final
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && apt-get install -y --no-install-recommends python3.12 && \
+    ln -s /usr/bin/python3.12 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set working directory
 WORKDIR /app
-COPY . /app/application
-ENV FLASK_APP=app.py
-ENV FLASK_DEBUG=true
 
+# Create a non-root user: `appuser` (Feel free to choose a name)
+RUN groupadd -r appuser && \
+    useradd -r -g appuser -d /app -s /sbin/nologin -c "Docker image user" appuser
+
+# Copy the virtual environment and model from the builder stage
+COPY --from=builder /venv /venv
+
+COPY --from=builder /models /app/models
+
+# Copy your application code
+COPY . /app/application
+
+# Change the ownership of the /app directory to the appuser
+	
+RUN mkdir -p /app/application/inputs/local
+RUN chown -R appuser:appuser /app
+
+# Set environment variables
+ENV FLASK_APP=app.py \
+    FLASK_DEBUG=true \
+    PATH="/venv/bin:$PATH"
+
+# Expose the port the app runs on
 EXPOSE 7091
 
-CMD ["gunicorn", "-w", "2", "--timeout", "120", "--bind", "0.0.0.0:7091", "application.wsgi:app"]
+# Switch to non-root user
+USER appuser
+
+# Start Gunicorn
+CMD ["gunicorn", "-w", "1", "--timeout", "120", "--bind", "0.0.0.0:7091", "--preload", "application.wsgi:app"]

application/agents/agent_creator.py

@@ -0,0 +1,16 @@
+from application.agents.classic_agent import ClassicAgent
+from application.agents.react_agent import ReActAgent
+
+
+class AgentCreator:
+    agents = {
+        "classic": ClassicAgent,
+        "react": ReActAgent,
+    }
+
+    @classmethod
+    def create_agent(cls, type, *args, **kwargs):
+        agent_class = cls.agents.get(type.lower())
+        if not agent_class:
+            raise ValueError(f"No agent class found for type {type}")
+        return agent_class(*args, **kwargs)

application/agents/base.py

@@ -0,0 +1,400 @@
+import logging
+import uuid
+from abc import ABC, abstractmethod
+from typing import Dict, Generator, List, Optional
+
+from bson.objectid import ObjectId
+
+from application.agents.tools.tool_action_parser import ToolActionParser
+from application.agents.tools.tool_manager import ToolManager
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.llm.handlers.handler_creator import LLMHandlerCreator
+from application.llm.llm_creator import LLMCreator
+from application.logging import build_stack_data, log_activity, LogContext
+from application.retriever.base import BaseRetriever
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAgent(ABC):
+    def __init__(
+        self,
+        endpoint: str,
+        llm_name: str,
+        gpt_model: str,
+        api_key: str,
+        user_api_key: Optional[str] = None,
+        prompt: str = "",
+        chat_history: Optional[List[Dict]] = None,
+        decoded_token: Optional[Dict] = None,
+        attachments: Optional[List[Dict]] = None,
+        json_schema: Optional[Dict] = None,
+    ):
+        self.endpoint = endpoint
+        self.llm_name = llm_name
+        self.gpt_model = gpt_model
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.prompt = prompt
+        self.decoded_token = decoded_token or {}
+        self.user: str = decoded_token.get("sub")
+        self.tool_config: Dict = {}
+        self.tools: List[Dict] = []
+        self.tool_calls: List[Dict] = []
+        self.chat_history: List[Dict] = chat_history if chat_history is not None else []
+        self.llm = LLMCreator.create_llm(
+            llm_name,
+            api_key=api_key,
+            user_api_key=user_api_key,
+            decoded_token=decoded_token,
+        )
+        self.llm_handler = LLMHandlerCreator.create_handler(
+            llm_name if llm_name else "default"
+        )
+        self.attachments = attachments or []
+        self.json_schema = json_schema
+
+    @log_activity()
+    def gen(
+        self, query: str, retriever: BaseRetriever, log_context: LogContext = None
+    ) -> Generator[Dict, None, None]:
+        yield from self._gen_inner(query, retriever, log_context)
+
+    @abstractmethod
+    def _gen_inner(
+        self, query: str, retriever: BaseRetriever, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        pass
+
+    def _get_tools(self, api_key: str = None) -> Dict[str, Dict]:
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        agents_collection = db["agents"]
+        tools_collection = db["user_tools"]
+
+        agent_data = agents_collection.find_one({"key": api_key or self.user_api_key})
+        tool_ids = agent_data.get("tools", []) if agent_data else []
+
+        tools = (
+            tools_collection.find(
+                {"_id": {"$in": [ObjectId(tool_id) for tool_id in tool_ids]}}
+            )
+            if tool_ids
+            else []
+        )
+        tools = list(tools)
+        tools_by_id = {str(tool["_id"]): tool for tool in tools} if tools else {}
+
+        return tools_by_id
+
+    def _get_user_tools(self, user="local"):
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        user_tools_collection = db["user_tools"]
+        user_tools = user_tools_collection.find({"user": user, "status": True})
+        user_tools = list(user_tools)
+
+        return {str(i): tool for i, tool in enumerate(user_tools)}
+
+    def _build_tool_parameters(self, action):
+        params = {"type": "object", "properties": {}, "required": []}
+        for param_type in ["query_params", "headers", "body", "parameters"]:
+            if param_type in action and action[param_type].get("properties"):
+                for k, v in action[param_type]["properties"].items():
+                    if v.get("filled_by_llm", True):
+                        params["properties"][k] = {
+                            key: value
+                            for key, value in v.items()
+                            if key != "filled_by_llm" and key != "value"
+                        }
+
+                        params["required"].append(k)
+        return params
+
+    def _prepare_tools(self, tools_dict):
+        self.tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": f"{action['name']}_{tool_id}",
+                    "description": action["description"],
+                    "parameters": self._build_tool_parameters(action),
+                },
+            }
+            for tool_id, tool in tools_dict.items()
+            if (
+                (tool["name"] == "api_tool" and "actions" in tool.get("config", {}))
+                or (tool["name"] != "api_tool" and "actions" in tool)
+            )
+            for action in (
+                tool["config"]["actions"].values()
+                if tool["name"] == "api_tool"
+                else tool["actions"]
+            )
+            if action.get("active", True)
+        ]
+
+    def _execute_tool_action(self, tools_dict, call):
+        parser = ToolActionParser(self.llm.__class__.__name__)
+        tool_id, action_name, call_args = parser.parse_args(call)
+
+        call_id = getattr(call, "id", None) or str(uuid.uuid4())
+        
+        # Check if parsing failed
+        if tool_id is None or action_name is None:
+            error_message = f"Error: Failed to parse LLM tool call. Tool name: {getattr(call, 'name', 'unknown')}"
+            logger.error(error_message)
+            
+            tool_call_data = {
+                "tool_name": "unknown",
+                "call_id": call_id,
+                "action_name": getattr(call, 'name', 'unknown'),
+                "arguments": call_args or {},
+                "result": f"Failed to parse tool call. Invalid tool name format: {getattr(call, 'name', 'unknown')}",
+            }
+            yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
+            self.tool_calls.append(tool_call_data)
+            return "Failed to parse tool call.", call_id
+        
+        # Check if tool_id exists in available tools
+        if tool_id not in tools_dict:
+            error_message = f"Error: Tool ID '{tool_id}' extracted from LLM call not found in available tools_dict. Available IDs: {list(tools_dict.keys())}"
+            logger.error(error_message)
+            
+            # Return error result
+            tool_call_data = {
+                "tool_name": "unknown",
+                "call_id": call_id,
+                "action_name": f"{action_name}_{tool_id}",
+                "arguments": call_args,
+                "result": f"Tool with ID {tool_id} not found. Available tools: {list(tools_dict.keys())}",
+            }
+            yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
+            self.tool_calls.append(tool_call_data)
+            return f"Tool with ID {tool_id} not found.", call_id
+        
+        tool_call_data = {
+            "tool_name": tools_dict[tool_id]["name"],
+            "call_id": call_id,
+            "action_name": f"{action_name}_{tool_id}",
+            "arguments": call_args,
+        }
+        yield {"type": "tool_call", "data": {**tool_call_data, "status": "pending"}}
+
+        tool_data = tools_dict[tool_id]
+        action_data = (
+            tool_data["config"]["actions"][action_name]
+            if tool_data["name"] == "api_tool"
+            else next(
+                action
+                for action in tool_data["actions"]
+                if action["name"] == action_name
+            )
+        )
+
+        query_params, headers, body, parameters = {}, {}, {}, {}
+        param_types = {
+            "query_params": query_params,
+            "headers": headers,
+            "body": body,
+            "parameters": parameters,
+        }
+
+        for param_type, target_dict in param_types.items():
+            if param_type in action_data and action_data[param_type].get("properties"):
+                for param, details in action_data[param_type]["properties"].items():
+                    if param not in call_args and "value" in details:
+                        target_dict[param] = details["value"]
+        for param, value in call_args.items():
+            for param_type, target_dict in param_types.items():
+                if param_type in action_data and param in action_data[param_type].get(
+                    "properties", {}
+                ):
+                    target_dict[param] = value
+        tm = ToolManager(config={})
+        tool = tm.load_tool(
+            tool_data["name"],
+            tool_config=(
+                {
+                    "url": tool_data["config"]["actions"][action_name]["url"],
+                    "method": tool_data["config"]["actions"][action_name]["method"],
+                    "headers": headers,
+                    "query_params": query_params,
+                }
+                if tool_data["name"] == "api_tool"
+                else tool_data["config"]
+            ),
+        )
+        if tool_data["name"] == "api_tool":
+            print(
+                f"Executing api: {action_name} with query_params: {query_params}, headers: {headers}, body: {body}"
+            )
+            result = tool.execute_action(action_name, **body)
+        else:
+            print(f"Executing tool: {action_name} with args: {call_args}")
+            result = tool.execute_action(action_name, **parameters)
+        tool_call_data["result"] = (
+            f"{str(result)[:50]}..." if len(str(result)) > 50 else result
+        )
+
+        yield {"type": "tool_call", "data": {**tool_call_data, "status": "completed"}}
+        self.tool_calls.append(tool_call_data)
+
+        return result, call_id
+
+    def _get_truncated_tool_calls(self):
+        return [
+            {
+                **tool_call,
+                "result": (
+                    f"{str(tool_call['result'])[:50]}..."
+                    if len(str(tool_call["result"])) > 50
+                    else tool_call["result"]
+                ),
+                "status": "completed",
+            }
+            for tool_call in self.tool_calls
+        ]
+
+    def _build_messages(
+        self,
+        system_prompt: str,
+        query: str,
+        retrieved_data: List[Dict],
+    ) -> List[Dict]:
+        docs_together = "\n".join([doc["text"] for doc in retrieved_data])
+        p_chat_combine = system_prompt.replace("{summaries}", docs_together)
+        messages_combine = [{"role": "system", "content": p_chat_combine}]
+
+        for i in self.chat_history:
+            if "prompt" in i and "response" in i:
+                messages_combine.append({"role": "user", "content": i["prompt"]})
+                messages_combine.append({"role": "assistant", "content": i["response"]})
+            if "tool_calls" in i:
+                for tool_call in i["tool_calls"]:
+                    call_id = tool_call.get("call_id") or str(uuid.uuid4())
+
+                    function_call_dict = {
+                        "function_call": {
+                            "name": tool_call.get("action_name"),
+                            "args": tool_call.get("arguments"),
+                            "call_id": call_id,
+                        }
+                    }
+                    function_response_dict = {
+                        "function_response": {
+                            "name": tool_call.get("action_name"),
+                            "response": {"result": tool_call.get("result")},
+                            "call_id": call_id,
+                        }
+                    }
+
+                    messages_combine.append(
+                        {"role": "assistant", "content": [function_call_dict]}
+                    )
+                    messages_combine.append(
+                        {"role": "tool", "content": [function_response_dict]}
+                    )
+        messages_combine.append({"role": "user", "content": query})
+        return messages_combine
+
+    def _retriever_search(
+        self,
+        retriever: BaseRetriever,
+        query: str,
+        log_context: Optional[LogContext] = None,
+    ) -> List[Dict]:
+        retrieved_data = retriever.search(query)
+        if log_context:
+            data = build_stack_data(retriever, exclude_attributes=["llm"])
+            log_context.stacks.append({"component": "retriever", "data": data})
+        return retrieved_data
+
+    def _llm_gen(self, messages: List[Dict], log_context: Optional[LogContext] = None):
+        gen_kwargs = {"model": self.gpt_model, "messages": messages}
+
+        if (
+            hasattr(self.llm, "_supports_tools")
+            and self.llm._supports_tools
+            and self.tools
+        ):
+            gen_kwargs["tools"] = self.tools
+
+        if (
+            self.json_schema
+            and hasattr(self.llm, "_supports_structured_output")
+            and self.llm._supports_structured_output()
+        ):
+            structured_format = self.llm.prepare_structured_output_format(
+                self.json_schema
+            )
+            if structured_format:
+                if self.llm_name == "openai":
+                    gen_kwargs["response_format"] = structured_format
+                elif self.llm_name == "google":
+                    gen_kwargs["response_schema"] = structured_format
+
+        resp = self.llm.gen_stream(**gen_kwargs)
+
+        if log_context:
+            data = build_stack_data(self.llm, exclude_attributes=["client"])
+            log_context.stacks.append({"component": "llm", "data": data})
+        return resp
+
+    def _llm_handler(
+        self,
+        resp,
+        tools_dict: Dict,
+        messages: List[Dict],
+        log_context: Optional[LogContext] = None,
+        attachments: Optional[List[Dict]] = None,
+    ):
+        resp = self.llm_handler.process_message_flow(
+            self, resp, tools_dict, messages, attachments, True
+        )
+        if log_context:
+            data = build_stack_data(self.llm_handler, exclude_attributes=["tool_calls"])
+            log_context.stacks.append({"component": "llm_handler", "data": data})
+        return resp
+
+    def _handle_response(self, response, tools_dict, messages, log_context):
+        is_structured_output = (
+            self.json_schema is not None
+            and hasattr(self.llm, "_supports_structured_output")
+            and self.llm._supports_structured_output()
+        )
+
+        if isinstance(response, str):
+            answer_data = {"answer": response}
+            if is_structured_output:
+                answer_data["structured"] = True
+                answer_data["schema"] = self.json_schema
+            yield answer_data
+            return
+        if hasattr(response, "message") and getattr(response.message, "content", None):
+            answer_data = {"answer": response.message.content}
+            if is_structured_output:
+                answer_data["structured"] = True
+                answer_data["schema"] = self.json_schema
+            yield answer_data
+            return
+        processed_response_gen = self._llm_handler(
+            response, tools_dict, messages, log_context, self.attachments
+        )
+
+        for event in processed_response_gen:
+            if isinstance(event, str):
+                answer_data = {"answer": event}
+                if is_structured_output:
+                    answer_data["structured"] = True
+                    answer_data["schema"] = self.json_schema
+                yield answer_data
+            elif hasattr(event, "message") and getattr(event.message, "content", None):
+                answer_data = {"answer": event.message.content}
+                if is_structured_output:
+                    answer_data["structured"] = True
+                    answer_data["schema"] = self.json_schema
+                yield answer_data
+            elif isinstance(event, dict) and "type" in event:
+                yield event

application/agents/classic_agent.py

@@ -0,0 +1,53 @@
+from typing import Dict, Generator
+from application.agents.base import BaseAgent
+from application.logging import LogContext
+from application.retriever.base import BaseRetriever
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ClassicAgent(BaseAgent):
+    """A simplified agent with clear execution flow.
+
+    Usage:
+    1. Processes a query through retrieval
+    2. Sets up available tools
+    3. Generates responses using LLM
+    4. Handles tool interactions if needed
+    5. Returns standardized outputs
+
+    Easy to extend by overriding specific steps.
+    """
+
+    def _gen_inner(
+        self, query: str, retriever: BaseRetriever, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        # Step 1: Retrieve relevant data
+        retrieved_data = self._retriever_search(retriever, query, log_context)
+
+        # Step 2: Prepare tools
+        tools_dict = (
+            self._get_user_tools(self.user)
+            if not self.user_api_key
+            else self._get_tools(self.user_api_key)
+        )
+        self._prepare_tools(tools_dict)
+
+        # Step 3: Build and process messages
+        messages = self._build_messages(self.prompt, query, retrieved_data)
+        llm_response = self._llm_gen(messages, log_context)
+
+        # Step 4: Handle the response
+        yield from self._handle_response(
+            llm_response, tools_dict, messages, log_context
+        )
+
+        # Step 5: Return metadata
+        yield {"sources": retrieved_data}
+        yield {"tool_calls": self._get_truncated_tool_calls()}
+
+        # Log tool calls for debugging
+        log_context.stacks.append(
+            {"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}}
+        )

application/agents/react_agent.py

@@ -0,0 +1,229 @@
+import os
+from typing import Dict, Generator, List, Any
+import logging
+
+from application.agents.base import BaseAgent
+from application.logging import build_stack_data, LogContext
+from application.retriever.base import BaseRetriever
+
+logger = logging.getLogger(__name__)
+
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+with open(
+    os.path.join(current_dir, "application/prompts", "react_planning_prompt.txt"), "r"
+) as f:
+    planning_prompt_template = f.read()
+with open(
+    os.path.join(current_dir, "application/prompts", "react_final_prompt.txt"),
+    "r",
+) as f:
+    final_prompt_template = f.read()
+    
+MAX_ITERATIONS_REASONING = 10
+
+class ReActAgent(BaseAgent):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.plan: str = ""
+        self.observations: List[str] = []
+
+    def _extract_content_from_llm_response(self, resp: Any) -> str:
+        """
+        Helper to extract string content from various LLM response types.
+        Handles strings, message objects (OpenAI-like), and streams.
+        Adapt stream handling for your specific LLM client if not OpenAI.
+        """
+        collected_content = []
+        if isinstance(resp, str):
+            collected_content.append(resp)
+        elif ( # OpenAI non-streaming or Anthropic non-streaming (older SDK style)
+            hasattr(resp, "message")
+            and hasattr(resp.message, "content")
+            and resp.message.content is not None
+        ):
+            collected_content.append(resp.message.content)
+        elif ( # OpenAI non-streaming (Pydantic model), Anthropic new SDK non-streaming
+            hasattr(resp, "choices") and resp.choices and
+            hasattr(resp.choices[0], "message") and
+            hasattr(resp.choices[0].message, "content") and
+            resp.choices[0].message.content is not None
+        ):
+            collected_content.append(resp.choices[0].message.content) # OpenAI
+        elif ( # Anthropic new SDK non-streaming content block
+             hasattr(resp, "content") and isinstance(resp.content, list) and resp.content and
+             hasattr(resp.content[0], "text")
+        ):
+            collected_content.append(resp.content[0].text) # Anthropic
+        else:
+            # Assume resp is a stream if not a recognized object
+            try:
+                for chunk in resp: # This will fail if resp is not iterable (e.g. a non-streaming response object)
+                    content_piece = ""
+                    # OpenAI-like stream
+                    if hasattr(chunk, 'choices') and len(chunk.choices) > 0 and \
+                       hasattr(chunk.choices[0], 'delta') and \
+                       hasattr(chunk.choices[0].delta, 'content') and \
+                       chunk.choices[0].delta.content is not None:
+                        content_piece = chunk.choices[0].delta.content
+                    # Anthropic-like stream (ContentBlockDelta)
+                    elif hasattr(chunk, 'type') and chunk.type == 'content_block_delta' and \
+                         hasattr(chunk, 'delta') and hasattr(chunk.delta, 'text'):
+                        content_piece = chunk.delta.text
+                    elif isinstance(chunk, str): # Simplest case: stream of strings
+                        content_piece = chunk
+
+                    if content_piece:
+                        collected_content.append(content_piece)
+            except TypeError: # If resp is not iterable (e.g. a final response object that wasn't caught above)
+                logger.debug(f"Response type {type(resp)} could not be iterated as a stream. It might be a non-streaming object not handled by specific checks.")
+            except Exception as e:
+                logger.error(f"Error processing potential stream chunk: {e}, chunk was: {getattr(chunk, '__dict__', chunk)}")
+
+
+        return "".join(collected_content)
+
+    def _gen_inner(
+        self, query: str, retriever: BaseRetriever, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        # Reset state for this generation call
+        self.plan = ""
+        self.observations = []
+        retrieved_data = self._retriever_search(retriever, query, log_context)
+
+        if self.user_api_key:
+            tools_dict = self._get_tools(self.user_api_key)
+        else:
+            tools_dict = self._get_user_tools(self.user)
+        self._prepare_tools(tools_dict)
+
+        docs_together = "\n".join([doc["text"] for doc in retrieved_data])
+        iterating_reasoning = 0
+        while iterating_reasoning < MAX_ITERATIONS_REASONING:
+            iterating_reasoning += 1
+            # 1. Create Plan
+            logger.info("ReActAgent: Creating plan...")
+            plan_stream = self._create_plan(query, docs_together, log_context)
+            current_plan_parts = []
+            yield {"thought": f"Reasoning... (iteration {iterating_reasoning})\n\n"}
+            for line_chunk in plan_stream:
+                current_plan_parts.append(line_chunk)
+                yield {"thought": line_chunk}
+            self.plan = "".join(current_plan_parts)
+            if self.plan:
+                self.observations.append(f"Plan: {self.plan} Iteration: {iterating_reasoning}")
+
+
+            max_obs_len = 20000
+            obs_str = "\n".join(self.observations)
+            if len(obs_str) > max_obs_len:
+                obs_str = obs_str[:max_obs_len] + "\n...[observations truncated]"
+            execution_prompt_str = (
+                (self.prompt or "")
+                + f"\n\nFollow this plan:\n{self.plan}"
+                + f"\n\nObservations:\n{obs_str}"
+                + f"\n\nIf there is enough data to complete user query '{query}', Respond with 'SATISFIED' only. Otherwise, continue. Dont Menstion 'SATISFIED' in your response if you are not ready. "
+            )
+            
+            messages = self._build_messages(execution_prompt_str, query, retrieved_data)
+
+            resp_from_llm_gen = self._llm_gen(messages, log_context)
+
+            initial_llm_thought_content = self._extract_content_from_llm_response(resp_from_llm_gen)
+            if initial_llm_thought_content:
+                self.observations.append(f"Initial thought/response: {initial_llm_thought_content}")
+            else:
+                logger.info("ReActAgent: Initial LLM response (before handler) had no textual content (might be only tool calls).")
+            resp_after_handler = self._llm_handler(resp_from_llm_gen, tools_dict, messages, log_context)
+            
+            for tool_call_info in self.tool_calls: # Iterate over self.tool_calls populated by _llm_handler
+                observation_string = (
+                    f"Executed Action: Tool '{tool_call_info.get('tool_name', 'N/A')}' "
+                    f"with arguments '{tool_call_info.get('arguments', '{}')}'. Result: '{str(tool_call_info.get('result', ''))[:200]}...'"
+                )
+                self.observations.append(observation_string)
+
+            content_after_handler = self._extract_content_from_llm_response(resp_after_handler)
+            if content_after_handler:
+                self.observations.append(f"Response after tool execution: {content_after_handler}")
+            else:
+                logger.info("ReActAgent: LLM response after handler had no textual content.")
+
+            if log_context:
+                log_context.stacks.append(
+                    {"component": "agent_tool_calls", "data": {"tool_calls": self.tool_calls.copy()}}
+                )
+
+            yield {"sources": retrieved_data}
+
+            display_tool_calls = []
+            for tc in self.tool_calls:
+                cleaned_tc = tc.copy()
+                if len(str(cleaned_tc.get("result", ""))) > 50:
+                    cleaned_tc["result"] = str(cleaned_tc["result"])[:50] + "..."
+                display_tool_calls.append(cleaned_tc)
+            if display_tool_calls:
+                yield {"tool_calls": display_tool_calls}
+            
+            if "SATISFIED" in content_after_handler:
+                logger.info("ReActAgent: LLM satisfied with the plan and data. Stopping reasoning.")
+                break
+
+        # 3. Create Final Answer based on all observations
+        final_answer_stream = self._create_final_answer(query, self.observations, log_context)
+        for answer_chunk in final_answer_stream:
+            yield {"answer": answer_chunk}
+        logger.info("ReActAgent: Finished generating final answer.")
+
+    def _create_plan(
+        self, query: str, docs_data: str, log_context: LogContext = None
+    ) -> Generator[str, None, None]:
+        plan_prompt_filled = planning_prompt_template.replace("{query}", query)
+        if "{summaries}" in plan_prompt_filled:
+            summaries = docs_data if docs_data else "No documents retrieved."
+            plan_prompt_filled = plan_prompt_filled.replace("{summaries}", summaries)
+        plan_prompt_filled = plan_prompt_filled.replace("{prompt}", self.prompt or "")
+        plan_prompt_filled = plan_prompt_filled.replace("{observations}", "\n".join(self.observations))
+
+        messages = [{"role": "user", "content": plan_prompt_filled}]
+
+        plan_stream_from_llm = self.llm.gen_stream(
+            model=self.gpt_model, messages=messages, tools=getattr(self, 'tools', None) # Use self.tools
+        )
+        if log_context:
+            data = build_stack_data(self.llm)
+            log_context.stacks.append({"component": "planning_llm", "data": data})
+
+        for chunk in plan_stream_from_llm:
+            content_piece = self._extract_content_from_llm_response(chunk)
+            if content_piece:
+                yield content_piece
+
+    def _create_final_answer(
+        self, query: str, observations: List[str], log_context: LogContext = None
+    ) -> Generator[str, None, None]:
+        observation_string = "\n".join(observations)
+        max_obs_len = 10000
+        if len(observation_string) > max_obs_len:
+            observation_string = observation_string[:max_obs_len] + "\n...[observations truncated]"
+            logger.warning("ReActAgent: Truncated observations for final answer prompt due to length.")
+
+        final_answer_prompt_filled = final_prompt_template.format(
+            query=query, observations=observation_string
+        )
+
+        messages = [{"role": "user", "content": final_answer_prompt_filled}]
+
+        # Final answer should synthesize, not call tools.
+        final_answer_stream_from_llm = self.llm.gen_stream(
+            model=self.gpt_model, messages=messages, tools=None
+        )
+        if log_context:
+            data = build_stack_data(self.llm)
+            log_context.stacks.append({"component": "final_answer_llm", "data": data})
+
+        for chunk in final_answer_stream_from_llm:
+            content_piece = self._extract_content_from_llm_response(chunk)
+            if content_piece:
+                yield content_piece

application/agents/tools/api_tool.py

@@ -0,0 +1,72 @@
+import json
+
+import requests
+from application.agents.tools.base import Tool
+
+
+class APITool(Tool):
+    """
+    API Tool
+    A flexible tool for performing various API actions (e.g., sending messages, retrieving data) via custom user-specified APIs
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.url = config.get("url", "")
+        self.method = config.get("method", "GET")
+        self.headers = config.get("headers", {"Content-Type": "application/json"})
+        self.query_params = config.get("query_params", {})
+
+    def execute_action(self, action_name, **kwargs):
+        return self._make_api_call(
+            self.url, self.method, self.headers, self.query_params, kwargs
+        )
+
+    def _make_api_call(self, url, method, headers, query_params, body):
+        if query_params:
+            url = f"{url}?{requests.compat.urlencode(query_params)}"
+        # if isinstance(body, dict):
+        #     body = json.dumps(body)
+        try:
+            print(f"Making API call: {method} {url} with body: {body}")
+            if body == "{}":
+                body = None
+            response = requests.request(method, url, headers=headers, data=body)
+            response.raise_for_status()
+            content_type = response.headers.get(
+                "Content-Type", "application/json"
+            ).lower()
+            if "application/json" in content_type:
+                try:
+                    data = response.json()
+                except json.JSONDecodeError as e:
+                    print(f"Error decoding JSON: {e}.  Raw response: {response.text}")
+                    return {
+                        "status_code": response.status_code,
+                        "message": f"API call returned invalid JSON.  Error: {e}",
+                        "data": response.text,
+                    }
+            elif "text/" in content_type or "application/xml" in content_type:
+                data = response.text
+            elif not response.content:
+                data = None
+            else:
+                print(f"Unsupported content type: {content_type}")
+                data = response.content
+
+            return {
+                "status_code": response.status_code,
+                "data": data,
+                "message": "API call successful.",
+            }
+        except requests.exceptions.RequestException as e:
+            return {
+                "status_code": response.status_code if response else None,
+                "message": f"API call failed: {str(e)}",
+            }
+
+    def get_actions_metadata(self):
+        return []
+
+    def get_config_requirements(self):
+        return {}

application/agents/tools/base.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+
+
+class Tool(ABC):
+    @abstractmethod
+    def execute_action(self, action_name: str, **kwargs):
+        pass
+
+    @abstractmethod
+    def get_actions_metadata(self):
+        """
+        Returns a list of JSON objects describing the actions supported by the tool.
+        """
+        pass
+
+    @abstractmethod
+    def get_config_requirements(self):
+        """
+        Returns a dictionary describing the configuration requirements for the tool.
+        """
+        pass

application/agents/tools/brave.py

@@ -0,0 +1,182 @@
+import requests
+from application.agents.tools.base import Tool
+
+
+class BraveSearchTool(Tool):
+    """
+    Brave Search
+    A tool for performing web and image searches using the Brave Search API.
+    Requires an API key for authentication.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.token = config.get("token", "")
+        self.base_url = "https://api.search.brave.com/res/v1"
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "brave_web_search": self._web_search,
+            "brave_image_search": self._image_search,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _web_search(
+        self,
+        query,
+        country="ALL",
+        search_lang="en",
+        count=10,
+        offset=0,
+        safesearch="off",
+        freshness=None,
+        result_filter=None,
+        extra_snippets=False,
+        summary=False,
+    ):
+        """
+        Performs a web search using the Brave Search API.
+        """
+        print(f"Performing Brave web search for: {query}")
+
+        url = f"{self.base_url}/web/search"
+
+        params = {
+            "q": query,
+            "country": country,
+            "search_lang": search_lang,
+            "count": min(count, 20),
+            "offset": min(offset, 9),
+            "safesearch": safesearch,
+        }
+
+        if freshness:
+            params["freshness"] = freshness
+        if result_filter:
+            params["result_filter"] = result_filter
+        if extra_snippets:
+            params["extra_snippets"] = 1
+        if summary:
+            params["summary"] = 1
+        headers = {
+            "Accept": "application/json",
+            "Accept-Encoding": "gzip",
+            "X-Subscription-Token": self.token,
+        }
+
+        response = requests.get(url, params=params, headers=headers)
+
+        if response.status_code == 200:
+            return {
+                "status_code": response.status_code,
+                "results": response.json(),
+                "message": "Search completed successfully.",
+            }
+        else:
+            return {
+                "status_code": response.status_code,
+                "message": f"Search failed with status code: {response.status_code}.",
+            }
+
+    def _image_search(
+        self,
+        query,
+        country="ALL",
+        search_lang="en",
+        count=5,
+        safesearch="off",
+        spellcheck=False,
+    ):
+        """
+        Performs an image search using the Brave Search API.
+        """
+        print(f"Performing Brave image search for: {query}")
+
+        url = f"{self.base_url}/images/search"
+
+        params = {
+            "q": query,
+            "country": country,
+            "search_lang": search_lang,
+            "count": min(count, 100),  # API max is 100
+            "safesearch": safesearch,
+            "spellcheck": 1 if spellcheck else 0,
+        }
+
+        headers = {
+            "Accept": "application/json",
+            "Accept-Encoding": "gzip",
+            "X-Subscription-Token": self.token,
+        }
+
+        response = requests.get(url, params=params, headers=headers)
+
+        if response.status_code == 200:
+            return {
+                "status_code": response.status_code,
+                "results": response.json(),
+                "message": "Image search completed successfully.",
+            }
+        else:
+            return {
+                "status_code": response.status_code,
+                "message": f"Image search failed with status code: {response.status_code}.",
+            }
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "brave_web_search",
+                "description": "Perform a web search using Brave Search",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "The search query (max 400 characters, 50 words)",
+                        },
+                        "search_lang": {
+                            "type": "string",
+                            "description": "The search language preference (default: en)",
+                        },
+                        "freshness": {
+                            "type": "string",
+                            "description": "Time filter for results (pd: last 24h, pw: last week, pm: last month, py: last year)",
+                        },
+                    },
+                    "required": ["query"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "brave_image_search",
+                "description": "Perform an image search using Brave Search",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "The search query (max 400 characters, 50 words)",
+                        },
+                        "count": {
+                            "type": "integer",
+                            "description": "Number of results to return (max 100, default: 5)",
+                        },
+                    },
+                    "required": ["query"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {
+                "type": "string",
+                "description": "Brave Search API key for authentication",
+            },
+        }

application/agents/tools/cryptoprice.py

@@ -0,0 +1,76 @@
+import requests
+from application.agents.tools.base import Tool
+
+
+class CryptoPriceTool(Tool):
+    """
+    CryptoPrice
+    A tool for retrieving cryptocurrency prices using the CryptoCompare public API
+    """
+
+    def __init__(self, config):
+        self.config = config
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {"cryptoprice_get": self._get_price}
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _get_price(self, symbol, currency):
+        """
+        Fetches the current price of a given cryptocurrency symbol in the specified currency.
+        Example:
+            symbol = "BTC"
+            currency = "USD"
+            returns price in USD.
+        """
+        url = f"https://min-api.cryptocompare.com/data/price?fsym={symbol.upper()}&tsyms={currency.upper()}"
+        response = requests.get(url)
+        if response.status_code == 200:
+            data = response.json()
+            if currency.upper() in data:
+                return {
+                    "status_code": response.status_code,
+                    "price": data[currency.upper()],
+                    "message": f"Price of {symbol.upper()} in {currency.upper()} retrieved successfully.",
+                }
+            else:
+                return {
+                    "status_code": response.status_code,
+                    "message": f"Couldn't find price for {symbol.upper()} in {currency.upper()}.",
+                }
+        else:
+            return {
+                "status_code": response.status_code,
+                "message": "Failed to retrieve price.",
+            }
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "cryptoprice_get",
+                "description": "Retrieve the price of a specified cryptocurrency in a given currency",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "symbol": {
+                            "type": "string",
+                            "description": "The cryptocurrency symbol (e.g. BTC)",
+                        },
+                        "currency": {
+                            "type": "string",
+                            "description": "The currency in which you want the price (e.g. USD)",
+                        },
+                    },
+                    "required": ["symbol", "currency"],
+                    "additionalProperties": False,
+                },
+            }
+        ]
+
+    def get_config_requirements(self):
+        # No specific configuration needed for this tool as it just queries a public endpoint
+        return {}

application/agents/tools/duckduckgo.py

@@ -0,0 +1,114 @@
+from application.agents.tools.base import Tool
+from duckduckgo_search import DDGS
+
+
+class DuckDuckGoSearchTool(Tool):
+    """
+    DuckDuckGo Search
+    A tool for performing web and image searches using DuckDuckGo.
+    """
+
+    def __init__(self, config):
+        self.config = config
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "ddg_web_search": self._web_search,
+            "ddg_image_search": self._image_search,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _web_search(
+        self,
+        query,
+        max_results=5,
+    ):
+        print(f"Performing DuckDuckGo web search for: {query}")
+
+        try:
+            results = DDGS().text(
+                query,
+                max_results=max_results,
+            )
+
+            return {
+                "status_code": 200,
+                "results": results,
+                "message": "Web search completed successfully.",
+            }
+        except Exception as e:
+            return {
+                "status_code": 500,
+                "message": f"Web search failed: {str(e)}",
+            }
+
+    def _image_search(
+        self,
+        query,
+        max_results=5,
+    ):
+        print(f"Performing DuckDuckGo image search for: {query}")
+
+        try:
+            results = DDGS().images(
+                keywords=query,
+                max_results=max_results,
+            )
+
+            return {
+                "status_code": 200,
+                "results": results,
+                "message": "Image search completed successfully.",
+            }
+        except Exception as e:
+            return {
+                "status_code": 500,
+                "message": f"Image search failed: {str(e)}",
+            }
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "ddg_web_search",
+                "description": "Perform a web search using DuckDuckGo.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "Search query",
+                        },
+                        "max_results": {
+                            "type": "integer",
+                            "description": "Number of results to return (default: 5)",
+                        },
+                    },
+                    "required": ["query"],
+                },
+            },
+            {
+                "name": "ddg_image_search",
+                "description": "Perform an image search using DuckDuckGo.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "Search query",
+                        },
+                        "max_results": {
+                            "type": "integer",
+                            "description": "Number of results to return (default: 5, max: 50)",
+                        },
+                    },
+                    "required": ["query"],
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {}

application/agents/tools/ntfy.py

@@ -0,0 +1,127 @@
+import requests
+from application.agents.tools.base import Tool
+
+class NtfyTool(Tool):
+    """
+    Ntfy Tool
+    A tool for sending notifications to ntfy topics on a specified server.
+    """
+
+    def __init__(self, config):
+        """
+        Initialize the NtfyTool with configuration.
+
+        Args:
+            config (dict): Configuration dictionary containing the access token.
+        """
+        self.config = config
+        self.token = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        """
+        Execute the specified action with given parameters.
+
+        Args:
+            action_name (str): Name of the action to execute.
+            **kwargs: Parameters for the action, including server_url.
+
+        Returns:
+            dict: Result of the action with status code and message.
+
+        Raises:
+            ValueError: If the action name is unknown.
+        """
+        actions = {
+            "ntfy_send_message": self._send_message,
+        }
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _send_message(self, server_url, message, topic, title=None, priority=None):
+        """
+        Send a message to an ntfy topic on the specified server.
+
+        Args:
+            server_url (str): Base URL of the ntfy server (e.g., https://ntfy.sh).
+            message (str): The message text to send.
+            topic (str): The topic to send the message to.
+            title (str, optional): Title of the notification.
+            priority (int, optional): Priority of the notification (1-5).
+
+        Returns:
+            dict: Response with status code and a confirmation message.
+
+        Raises:
+            ValueError: If priority is not an integer between 1 and 5.
+        """
+        url = f"{server_url.rstrip('/')}/{topic}"
+        headers = {}
+        if title:
+            headers["X-Title"] = title
+        if priority:
+            try:
+                priority = int(priority)
+            except (ValueError, TypeError):
+                raise ValueError("Priority must be convertible to an integer")
+            if priority < 1 or priority > 5:
+                raise ValueError("Priority must be an integer between 1 and 5")
+            headers["X-Priority"] = str(priority)
+        if self.token:
+            headers["Authorization"] = f"Basic {self.token}"
+        data = message.encode("utf-8")
+        response = requests.post(url, headers=headers, data=data)
+        return {"status_code": response.status_code, "message": "Message sent"}
+
+    def get_actions_metadata(self):
+        """
+        Provide metadata about available actions.
+
+        Returns:
+            list: List of dictionaries describing each action.
+        """
+        return [
+            {
+                "name": "ntfy_send_message",
+                "description": "Send a notification to an ntfy topic",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "server_url": {
+                            "type": "string",
+                            "description": "Base URL of the ntfy server",
+                        },
+                        "message": {
+                            "type": "string",
+                            "description": "Text to send in the notification",
+                        },
+                        "topic": {
+                            "type": "string",
+                            "description": "Topic to send the notification to",
+                        },
+                        "title": {
+                            "type": "string",
+                            "description": "Title of the notification (optional)",
+                        },
+                        "priority": {
+                            "type": "integer",
+                            "description": "Priority of the notification (1-5, optional)",
+                        },
+                    },
+                    "required": ["server_url", "message", "topic"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        """
+        Specify the configuration requirements.
+
+        Returns:
+            dict: Dictionary describing required config parameters.
+        """
+        return {
+            "token": {"type": "string", "description": "Access token for authentication"},
+        }

application/agents/tools/postgres.py

@@ -0,0 +1,163 @@
+import psycopg2
+from application.agents.tools.base import Tool
+
+class PostgresTool(Tool):
+    """
+    PostgreSQL Database Tool
+    A tool for connecting to a PostgreSQL database using a connection string,
+    executing SQL queries, and retrieving schema information.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.connection_string = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "postgres_execute_sql": self._execute_sql,
+            "postgres_get_schema": self._get_schema,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _execute_sql(self, sql_query):
+        """
+        Executes an SQL query against the PostgreSQL database using a connection string.
+        """
+        conn = None  # Initialize conn to None for error handling
+        try:
+            conn = psycopg2.connect(self.connection_string)
+            cur = conn.cursor()
+            cur.execute(sql_query)
+            conn.commit()
+
+            if sql_query.strip().lower().startswith("select"):
+                column_names = [desc[0] for desc in cur.description] if cur.description else []
+                results = []
+                rows = cur.fetchall()
+                for row in rows:
+                    results.append(dict(zip(column_names, row)))
+                response_data = {"data": results, "column_names": column_names}
+            else:
+                row_count = cur.rowcount
+                response_data = {"message": f"Query executed successfully, {row_count} rows affected."}
+
+            cur.close()
+            return {
+                "status_code": 200,
+                "message": "SQL query executed successfully.",
+                "response_data": response_data,
+            }
+
+        except psycopg2.Error as e:
+            error_message = f"Database error: {e}"
+            print(f"Database error: {e}")
+            return {
+                "status_code": 500,
+                "message": "Failed to execute SQL query.",
+                "error": error_message,
+            }
+        finally:
+            if conn:  # Ensure connection is closed even if errors occur
+                conn.close()
+
+    def _get_schema(self, db_name):
+        """
+        Retrieves the schema of the PostgreSQL database using a connection string.
+        """
+        conn = None # Initialize conn to None for error handling
+        try:
+            conn = psycopg2.connect(self.connection_string)
+            cur = conn.cursor()
+
+            cur.execute("""
+                SELECT
+                    table_name,
+                    column_name,
+                    data_type,
+                    column_default,
+                    is_nullable
+                FROM
+                    information_schema.columns
+                WHERE
+                    table_schema = 'public'
+                ORDER BY
+                    table_name,
+                    ordinal_position;
+            """)
+
+            schema_data = {}
+            for row in cur.fetchall():
+                table_name, column_name, data_type, column_default, is_nullable = row
+                if table_name not in schema_data:
+                    schema_data[table_name] = []
+                schema_data[table_name].append({
+                    "column_name": column_name,
+                    "data_type": data_type,
+                    "column_default": column_default,
+                    "is_nullable": is_nullable
+                })
+
+            cur.close()
+            return {
+                "status_code": 200,
+                "message": "Database schema retrieved successfully.",
+                "schema": schema_data,
+            }
+
+        except psycopg2.Error as e:
+            error_message = f"Database error: {e}"
+            print(f"Database error: {e}")
+            return {
+                "status_code": 500,
+                "message": "Failed to retrieve database schema.",
+                "error": error_message,
+            }
+        finally:
+            if conn: # Ensure connection is closed even if errors occur
+                conn.close()
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "postgres_execute_sql",
+                "description": "Execute an SQL query against the PostgreSQL database and return the results. Use this tool to interact with the database, e.g., retrieve specific data or perform updates. Only SELECT queries will return data, other queries will return execution status.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "sql_query": {
+                            "type": "string",
+                            "description": "The SQL query to execute.",
+                        },
+                    },
+                    "required": ["sql_query"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "postgres_get_schema",
+                "description": "Retrieve the schema of the PostgreSQL database, including tables and their columns. Use this to understand the database structure before executing queries. db_name is 'default' if not provided.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "db_name": {
+                            "type": "string",
+                            "description": "The name of the database to retrieve the schema for.",
+                        },
+                    },
+                    "required": ["db_name"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {
+                "type": "string",
+                "description": "PostgreSQL database connection string (e.g., 'postgresql://user:password@host:port/dbname')",
+            },
+        }

application/agents/tools/read_webpage.py

@@ -0,0 +1,83 @@
+import requests
+from markdownify import markdownify
+from application.agents.tools.base import Tool
+from urllib.parse import urlparse
+
+class ReadWebpageTool(Tool):
+    """
+    Read Webpage (browser)
+    A tool to fetch the HTML content of a URL and convert it to Markdown.
+    """
+
+    def __init__(self, config=None):
+        """
+        Initializes the tool.
+        :param config: Optional configuration dictionary. Not used by this tool.
+        """
+        self.config = config
+
+    def execute_action(self, action_name: str, **kwargs) -> str:
+        """
+        Executes the specified action. For this tool, the only action is 'read_webpage'.
+
+        :param action_name: The name of the action to execute. Should be 'read_webpage'.
+        :param kwargs: Keyword arguments, must include 'url'.
+        :return: The Markdown content of the webpage or an error message.
+        """
+        if action_name != "read_webpage":
+            return f"Error: Unknown action '{action_name}'. This tool only supports 'read_webpage'."
+
+        url = kwargs.get("url")
+        if not url:
+            return "Error: URL parameter is missing."
+
+        # Ensure the URL has a scheme (if not, default to http)
+        parsed_url = urlparse(url)
+        if not parsed_url.scheme:
+            url = "http://" + url
+        
+        try:
+            response = requests.get(url, timeout=10, headers={'User-Agent': 'DocsGPT-Agent/1.0'})
+            response.raise_for_status()  # Raise an exception for HTTP errors (4xx or 5xx)
+            
+            html_content = response.text
+            #soup = BeautifulSoup(html_content, 'html.parser')
+            
+            
+            markdown_content = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
+            
+            return markdown_content
+
+        except requests.exceptions.RequestException as e:
+            return f"Error fetching URL {url}: {e}"
+        except Exception as e:
+            return f"Error processing URL {url}: {e}"
+
+    def get_actions_metadata(self):
+        """
+        Returns metadata for the actions supported by this tool.
+        """
+        return [
+            {
+                "name": "read_webpage",
+                "description": "Fetches the HTML content of a given URL and returns it as clean Markdown text. Input must be a valid URL.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "url": {
+                            "type": "string",
+                            "description": "The fully qualified URL of the webpage to read (e.g., 'https://www.example.com').",
+                        }
+                    },
+                    "required": ["url"],
+                    "additionalProperties": False,
+                },
+            }
+        ]
+
+    def get_config_requirements(self):
+        """
+        Returns a dictionary describing the configuration requirements for the tool.
+        This tool does not require any specific configuration.
+        """
+        return {}

application/agents/tools/telegram.py

@@ -0,0 +1,86 @@
+import requests
+from application.agents.tools.base import Tool
+
+
+class TelegramTool(Tool):
+    """
+    Telegram Bot
+    A flexible Telegram tool for performing various actions (e.g., sending messages, images).
+    Requires a bot token and chat ID for configuration
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.token = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "telegram_send_message": self._send_message,
+            "telegram_send_image": self._send_image,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _send_message(self, text, chat_id):
+        print(f"Sending message: {text}")
+        url = f"https://api.telegram.org/bot{self.token}/sendMessage"
+        payload = {"chat_id": chat_id, "text": text}
+        response = requests.post(url, data=payload)
+        return {"status_code": response.status_code, "message": "Message sent"}
+
+    def _send_image(self, image_url, chat_id):
+        print(f"Sending image: {image_url}")
+        url = f"https://api.telegram.org/bot{self.token}/sendPhoto"
+        payload = {"chat_id": chat_id, "photo": image_url}
+        response = requests.post(url, data=payload)
+        return {"status_code": response.status_code, "message": "Image sent"}
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "telegram_send_message",
+                "description": "Send a notification to Telegram chat",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "text": {
+                            "type": "string",
+                            "description": "Text to send in the notification",
+                        },
+                        "chat_id": {
+                            "type": "string",
+                            "description": "Chat ID to send the notification to",
+                        },
+                    },
+                    "required": ["text"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "telegram_send_image",
+                "description": "Send an image to the Telegram chat",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "image_url": {
+                            "type": "string",
+                            "description": "URL of the image to send",
+                        },
+                        "chat_id": {
+                            "type": "string",
+                            "description": "Chat ID to send the image to",
+                        },
+                    },
+                    "required": ["image_url"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {"type": "string", "description": "Bot token for authentication"},
+        }

application/agents/tools/tool_action_parser.py

@@ -0,0 +1,61 @@
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ToolActionParser:
+    def __init__(self, llm_type):
+        self.llm_type = llm_type
+        self.parsers = {
+            "OpenAILLM": self._parse_openai_llm,
+            "GoogleLLM": self._parse_google_llm,
+        }
+
+    def parse_args(self, call):
+        parser = self.parsers.get(self.llm_type, self._parse_openai_llm)
+        return parser(call)
+
+    def _parse_openai_llm(self, call):
+        try:
+            call_args = json.loads(call.arguments)
+            tool_parts = call.name.split("_")
+            
+            # If the tool name doesn't contain an underscore, it's likely a hallucinated tool
+            if len(tool_parts) < 2:
+                logger.warning(f"Invalid tool name format: {call.name}. Expected format: action_name_tool_id")
+                return None, None, None
+                
+            tool_id = tool_parts[-1]
+            action_name = "_".join(tool_parts[:-1])
+            
+            # Validate that tool_id looks like a numerical ID
+            if not tool_id.isdigit():
+                logger.warning(f"Tool ID '{tool_id}' is not numerical. This might be a hallucinated tool call.")
+                
+        except (AttributeError, TypeError) as e:
+            logger.error(f"Error parsing OpenAI LLM call: {e}")
+            return None, None, None
+        return tool_id, action_name, call_args
+
+    def _parse_google_llm(self, call):
+        try:
+            call_args = call.arguments
+            tool_parts = call.name.split("_")
+            
+            # If the tool name doesn't contain an underscore, it's likely a hallucinated tool
+            if len(tool_parts) < 2:
+                logger.warning(f"Invalid tool name format: {call.name}. Expected format: action_name_tool_id")
+                return None, None, None
+                
+            tool_id = tool_parts[-1]
+            action_name = "_".join(tool_parts[:-1])
+            
+            # Validate that tool_id looks like a numerical ID
+            if not tool_id.isdigit():
+                logger.warning(f"Tool ID '{tool_id}' is not numerical. This might be a hallucinated tool call.")
+                
+        except (AttributeError, TypeError) as e:
+            logger.error(f"Error parsing Google LLM call: {e}")
+            return None, None, None
+        return tool_id, action_name, call_args

application/agents/tools/tool_manager.py

@@ -0,0 +1,42 @@
+import importlib
+import inspect
+import os
+import pkgutil
+
+from application.agents.tools.base import Tool
+
+
+class ToolManager:
+    def __init__(self, config):
+        self.config = config
+        self.tools = {}
+        self.load_tools()
+
+    def load_tools(self):
+        tools_dir = os.path.join(os.path.dirname(__file__))
+        for finder, name, ispkg in pkgutil.iter_modules([tools_dir]):
+            if name == "base" or name.startswith("__"):
+                continue
+            module = importlib.import_module(f"application.agents.tools.{name}")
+            for member_name, obj in inspect.getmembers(module, inspect.isclass):
+                if issubclass(obj, Tool) and obj is not Tool:
+                    tool_config = self.config.get(name, {})
+                    self.tools[name] = obj(tool_config)
+
+    def load_tool(self, tool_name, tool_config):
+        self.config[tool_name] = tool_config
+        module = importlib.import_module(f"application.agents.tools.{tool_name}")
+        for member_name, obj in inspect.getmembers(module, inspect.isclass):
+            if issubclass(obj, Tool) and obj is not Tool:
+                return obj(tool_config)
+
+    def execute_action(self, tool_name, action_name, **kwargs):
+        if tool_name not in self.tools:
+            raise ValueError(f"Tool '{tool_name}' not loaded")
+        return self.tools[tool_name].execute_action(action_name, **kwargs)
+
+    def get_all_actions_metadata(self):
+        metadata = []
+        for tool in self.tools.values():
+            metadata.extend(tool.get_actions_metadata())
+        return metadata

application/api/__init__.py

@@ -0,0 +1,7 @@
+from flask_restx import Api
+
+api = Api(
+    version="1.0",
+    title="DocsGPT API",
+    description="API for DocsGPT",
+)

application/api/answer/__init__.py

@@ -0,0 +1,19 @@
+from flask import Blueprint
+
+from application.api import api
+from application.api.answer.routes.answer import AnswerResource
+from application.api.answer.routes.base import answer_ns
+from application.api.answer.routes.stream import StreamResource
+
+
+answer = Blueprint("answer", __name__)
+
+api.add_namespace(answer_ns)
+
+
+def init_answer_routes():
+    api.add_resource(StreamResource, "/stream")
+    api.add_resource(AnswerResource, "/api/answer")
+
+
+init_answer_routes()

application/api/answer/routes.py

@@ -1,337 +0,0 @@
-import asyncio
-import os
-from flask import Blueprint, request, Response
-import json
-import datetime
-import logging
-import traceback
-
-from pymongo import MongoClient
-from bson.objectid import ObjectId
-from transformers import GPT2TokenizerFast
-
-
-
-from application.core.settings import settings
-from application.vectorstore.vector_creator import VectorCreator
-from application.llm.llm_creator import LLMCreator
-from application.error import bad_request
-
-
-
-logger = logging.getLogger(__name__)
-
-mongo = MongoClient(settings.MONGO_URI)
-db = mongo["docsgpt"]
-conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
-answer = Blueprint('answer', __name__)
-
-if settings.LLM_NAME == "gpt4":
-    gpt_model = 'gpt-4'
-else:
-    gpt_model = 'gpt-3.5-turbo'
-
-# load the prompts
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-with open(os.path.join(current_dir, "prompts", "combine_prompt.txt"), "r") as f:
-    template = f.read()
-
-with open(os.path.join(current_dir, "prompts", "combine_prompt_hist.txt"), "r") as f:
-    template_hist = f.read()
-
-with open(os.path.join(current_dir, "prompts", "question_prompt.txt"), "r") as f:
-    template_quest = f.read()
-
-with open(os.path.join(current_dir, "prompts", "chat_combine_prompt.txt"), "r") as f:
-    chat_combine_template = f.read()
-
-with open(os.path.join(current_dir, "prompts", "chat_reduce_prompt.txt"), "r") as f:
-    chat_reduce_template = f.read()
-
-api_key_set = settings.API_KEY is not None
-embeddings_key_set = settings.EMBEDDINGS_KEY is not None
-
-
-async def async_generate(chain, question, chat_history):
-    result = await chain.arun({"question": question, "chat_history": chat_history})
-    return result
-
-
-def count_tokens(string):
-    tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
-    return len(tokenizer(string)['input_ids'])
-
-
-def run_async_chain(chain, question, chat_history):
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-    result = {}
-    try:
-        answer = loop.run_until_complete(async_generate(chain, question, chat_history))
-    finally:
-        loop.close()
-    result["answer"] = answer
-    return result
-
-
-def get_vectorstore(data):
-    if "active_docs" in data:
-        if data["active_docs"].split("/")[0] == "local":
-            if data["active_docs"].split("/")[1] == "default":
-                vectorstore = ""
-            else:
-                vectorstore = "indexes/" + data["active_docs"]
-        else:
-            vectorstore = "vectors/" + data["active_docs"]
-        if data["active_docs"] == "default":
-            vectorstore = ""
-    else:
-        vectorstore = ""
-    vectorstore = os.path.join("application", vectorstore)
-    return vectorstore
-
-
-# def get_docsearch(vectorstore, embeddings_key):
-#     if settings.EMBEDDINGS_NAME == "openai_text-embedding-ada-002":
-#         if is_azure_configured():
-#             os.environ["OPENAI_API_TYPE"] = "azure"
-#             openai_embeddings = OpenAIEmbeddings(model=settings.AZURE_EMBEDDINGS_DEPLOYMENT_NAME)
-#         else:
-#             openai_embeddings = OpenAIEmbeddings(openai_api_key=embeddings_key)
-#         docsearch = FAISS.load_local(vectorstore, openai_embeddings)
-#     elif settings.EMBEDDINGS_NAME == "huggingface_sentence-transformers/all-mpnet-base-v2":
-#         docsearch = FAISS.load_local(vectorstore, HuggingFaceHubEmbeddings())
-#     elif settings.EMBEDDINGS_NAME == "huggingface_hkunlp/instructor-large":
-#         docsearch = FAISS.load_local(vectorstore, HuggingFaceInstructEmbeddings())
-#     elif settings.EMBEDDINGS_NAME == "cohere_medium":
-#         docsearch = FAISS.load_local(vectorstore, CohereEmbeddings(cohere_api_key=embeddings_key))
-#     return docsearch
-
-
-def is_azure_configured():
-    return settings.OPENAI_API_BASE and settings.OPENAI_API_VERSION and settings.AZURE_DEPLOYMENT_NAME
-
-
-def complete_stream(question, docsearch, chat_history, api_key, conversation_id):
-    llm = LLMCreator.create_llm(settings.LLM_NAME, api_key=api_key)
-    
-
-    docs = docsearch.search(question, k=2)
-    if settings.LLM_NAME == "llama.cpp":
-        docs = [docs[0]]
-    # join all page_content together with a newline
-    docs_together = "\n".join([doc.page_content for doc in docs])
-    p_chat_combine = chat_combine_template.replace("{summaries}", docs_together)
-    messages_combine = [{"role": "system", "content": p_chat_combine}]
-    source_log_docs = []
-    for doc in docs:
-        if doc.metadata:
-            data = json.dumps({"type": "source", "doc": doc.page_content, "metadata": doc.metadata})
-            source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content})
-        else:
-            data = json.dumps({"type": "source", "doc": doc.page_content})
-            source_log_docs.append({"title": doc.page_content, "text": doc.page_content})
-        yield f"data:{data}\n\n"
-
-    if len(chat_history) > 1:
-        tokens_current_history = 0
-        # count tokens in history
-        chat_history.reverse()
-        for i in chat_history:
-            if "prompt" in i and "response" in i:
-                tokens_batch = count_tokens(i["prompt"]) + count_tokens(i["response"])
-                if tokens_current_history + tokens_batch < settings.TOKENS_MAX_HISTORY:
-                    tokens_current_history += tokens_batch
-                    messages_combine.append({"role": "user", "content": i["prompt"]})
-                    messages_combine.append({"role": "system", "content": i["response"]})
-    messages_combine.append({"role": "user", "content": question})
-
-    response_full = ""
-    completion = llm.gen_stream(model=gpt_model, engine=settings.AZURE_DEPLOYMENT_NAME,
-                                messages=messages_combine)
-    for line in completion:
-        data = json.dumps({"answer": str(line)})
-        response_full += str(line)
-        yield f"data: {data}\n\n"
-
-    # save conversation to database
-    if conversation_id is not None:
-        conversations_collection.update_one(
-            {"_id": ObjectId(conversation_id)},
-            {"$push": {"queries": {"prompt": question, "response": response_full, "sources": source_log_docs}}},
-        )
-
-    else:
-        # create new conversation
-        # generate summary
-        messages_summary = [{"role": "assistant", "content": "Summarise following conversation in no more than 3 "
-                                                             "words, respond ONLY with the summary, use the same "
-                                                             "language as the system \n\nUser: " + question + "\n\n" +
-                                                             "AI: " +
-                                                             response_full},
-                            {"role": "user", "content": "Summarise following conversation in no more than 3 words, "
-                                                        "respond ONLY with the summary, use the same language as the "
-                                                        "system"}]
-
-        completion = llm.gen(model=gpt_model, engine=settings.AZURE_DEPLOYMENT_NAME,
-                             messages=messages_summary, max_tokens=30)
-        conversation_id = conversations_collection.insert_one(
-            {"user": "local",
-             "date": datetime.datetime.utcnow(),
-             "name": completion,
-             "queries": [{"prompt": question, "response": response_full, "sources": source_log_docs}]}
-        ).inserted_id
-
-    # send data.type = "end" to indicate that the stream has ended as json
-    data = json.dumps({"type": "id", "id": str(conversation_id)})
-    yield f"data: {data}\n\n"
-    data = json.dumps({"type": "end"})
-    yield f"data: {data}\n\n"
-
-
-@answer.route("/stream", methods=["POST"])
-def stream():
-    data = request.get_json()
-    # get parameter from url question
-    question = data["question"]
-    history = data["history"]
-    # history to json object from string
-    history = json.loads(history)
-    conversation_id = data["conversation_id"]
-
-    # check if active_docs is set
-
-    if not api_key_set:
-        api_key = data["api_key"]
-    else:
-        api_key = settings.API_KEY
-    if not embeddings_key_set:
-        embeddings_key = data["embeddings_key"]
-    else:
-        embeddings_key = settings.EMBEDDINGS_KEY
-    if "active_docs" in data:
-        vectorstore = get_vectorstore({"active_docs": data["active_docs"]})
-    else:
-        vectorstore = ""
-    docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key)
-
-    return Response(
-        complete_stream(question, docsearch,
-                        chat_history=history, api_key=api_key,
-                        conversation_id=conversation_id), mimetype="text/event-stream"
-    )
-
-
-@answer.route("/api/answer", methods=["POST"])
-def api_answer():
-    data = request.get_json()
-    question = data["question"]
-    history = data["history"]
-    if "conversation_id" not in data:
-        conversation_id = None
-    else:
-        conversation_id = data["conversation_id"]
-    print("-" * 5)
-    if not api_key_set:
-        api_key = data["api_key"]
-    else:
-        api_key = settings.API_KEY
-    if not embeddings_key_set:
-        embeddings_key = data["embeddings_key"]
-    else:
-        embeddings_key = settings.EMBEDDINGS_KEY
-
-    # use try and except  to check for exception
-    try:
-        # check if the vectorstore is set
-        vectorstore = get_vectorstore(data)
-        # loading the index and the store and the prompt template
-        # Note if you have used other embeddings than OpenAI, you need to change the embeddings
-        docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key)
-
-
-        llm = LLMCreator.create_llm(settings.LLM_NAME, api_key=api_key)
-
-
-
-        docs = docsearch.search(question, k=2)
-        # join all page_content together with a newline
-        docs_together = "\n".join([doc.page_content for doc in docs])
-        p_chat_combine = chat_combine_template.replace("{summaries}", docs_together)
-        messages_combine = [{"role": "system", "content": p_chat_combine}]
-        source_log_docs = []
-        for doc in docs:
-            if doc.metadata:
-                source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content})
-            else:
-                source_log_docs.append({"title": doc.page_content, "text": doc.page_content})
-        # join all page_content together with a newline
-
-
-        if len(history) > 1:
-            tokens_current_history = 0
-            # count tokens in history
-            history.reverse()
-            for i in history:
-                if "prompt" in i and "response" in i:
-                    tokens_batch = count_tokens(i["prompt"]) + count_tokens(i["response"])
-                    if tokens_current_history + tokens_batch < settings.TOKENS_MAX_HISTORY:
-                        tokens_current_history += tokens_batch
-                        messages_combine.append({"role": "user", "content": i["prompt"]})
-                        messages_combine.append({"role": "system", "content": i["response"]})
-        messages_combine.append({"role": "user", "content": question})
-
-
-        completion = llm.gen(model=gpt_model, engine=settings.AZURE_DEPLOYMENT_NAME,
-                                    messages=messages_combine)
-
-
-        result = {"answer": completion, "sources": source_log_docs}
-        logger.debug(result)
-
-        # generate conversationId
-        if conversation_id is not None:
-            conversations_collection.update_one(
-                {"_id": ObjectId(conversation_id)},
-                {"$push": {"queries": {"prompt": question,
-                                       "response": result["answer"], "sources": result['sources']}}},
-            )
-
-        else:
-            # create new conversation
-            # generate summary
-            messages_summary = [
-                {"role": "assistant", "content": "Summarise following conversation in no more than 3 words, "
-                    "respond ONLY with the summary, use the same language as the system \n\n"
-                    "User: " + question + "\n\n" + "AI: " + result["answer"]},
-                {"role": "user", "content": "Summarise following conversation in no more than 3 words, "
-                    "respond ONLY with the summary, use the same language as the system"}
-            ]
-
-            completion = llm.gen(
-                model=gpt_model,
-                engine=settings.AZURE_DEPLOYMENT_NAME,
-                messages=messages_summary,
-                max_tokens=30
-            )
-            conversation_id = conversations_collection.insert_one(
-                {"user": "local",
-                "date": datetime.datetime.utcnow(),
-                "name": completion,
-                "queries": [{"prompt": question, "response": result["answer"], "sources": source_log_docs}]}
-            ).inserted_id
-
-        result["conversation_id"] = str(conversation_id)
-
-        # mock result
-        # result = {
-        #     "answer": "The answer is 42",
-        #     "sources": ["https://en.wikipedia.org/wiki/42_(number)", "https://en.wikipedia.org/wiki/42_(number)"]
-        # }
-        return result
-    except Exception as e:
-        # print whole traceback
-        traceback.print_exc()
-        print(str(e))
-        return bad_request(500, str(e))

application/api/answer/routes/answer.py

@@ -0,0 +1,122 @@
+import logging
+import traceback
+
+from flask import make_response, request
+from flask_restx import fields, Resource
+
+from application.api import api
+
+from application.api.answer.routes.base import answer_ns, BaseAnswerResource
+
+from application.api.answer.services.stream_processor import StreamProcessor
+
+logger = logging.getLogger(__name__)
+
+
+@answer_ns.route("/api/answer")
+class AnswerResource(Resource, BaseAnswerResource):
+    def __init__(self, *args, **kwargs):
+        Resource.__init__(self, *args, **kwargs)
+        BaseAnswerResource.__init__(self)
+
+    answer_model = answer_ns.model(
+        "AnswerModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
+            ),
+            "history": fields.List(
+                fields.String,
+                required=False,
+                description="Conversation history (only for new conversations)",
+            ),
+            "conversation_id": fields.String(
+                required=False,
+                description="Existing conversation ID (loads history)",
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+            "save_conversation": fields.Boolean(
+                required=False,
+                default=True,
+                description="Whether to save the conversation",
+            ),
+        },
+    )
+
+    @api.expect(answer_model)
+    @api.doc(description="Provide a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        if error := self.validate_request(data):
+            return error
+        decoded_token = getattr(request, "decoded_token", None)
+        processor = StreamProcessor(data, decoded_token)
+        try:
+            processor.initialize()
+            if not processor.decoded_token:
+                return make_response({"error": "Unauthorized"}, 401)
+            agent = processor.create_agent()
+            retriever = processor.create_retriever()
+
+            stream = self.complete_stream(
+                question=data["question"],
+                agent=agent,
+                retriever=retriever,
+                conversation_id=processor.conversation_id,
+                user_api_key=processor.agent_config.get("user_api_key"),
+                decoded_token=processor.decoded_token,
+                isNoneDoc=data.get("isNoneDoc"),
+                index=None,
+                should_save_conversation=data.get("save_conversation", True),
+            )
+            stream_result = self.process_response_stream(stream)
+
+            if len(stream_result) == 7:
+                (
+                    conversation_id,
+                    response,
+                    sources,
+                    tool_calls,
+                    thought,
+                    error,
+                    structured_info,
+                ) = stream_result
+            else:
+                conversation_id, response, sources, tool_calls, thought, error = (
+                    stream_result
+                )
+                structured_info = None
+
+            if error:
+                return make_response({"error": error}, 400)
+            result = {
+                "conversation_id": conversation_id,
+                "answer": response,
+                "sources": sources,
+                "tool_calls": tool_calls,
+                "thought": thought,
+            }
+
+            if structured_info:
+                result.update(structured_info)
+        except Exception as e:
+            logger.error(
+                f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return make_response({"error": str(e)}, 500)
+        return make_response(result, 200)

application/api/answer/routes/base.py

@@ -0,0 +1,263 @@
+import datetime
+import json
+import logging
+from typing import Any, Dict, Generator, List, Optional
+
+from flask import Response
+from flask_restx import Namespace
+
+from application.api.answer.services.conversation_service import ConversationService
+
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.llm.llm_creator import LLMCreator
+from application.utils import check_required_fields, get_gpt_model
+
+logger = logging.getLogger(__name__)
+
+
+answer_ns = Namespace("answer", description="Answer related operations", path="/")
+
+
+class BaseAnswerResource:
+    """Shared base class for answer endpoints"""
+
+    def __init__(self):
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        self.user_logs_collection = db["user_logs"]
+        self.gpt_model = get_gpt_model()
+        self.conversation_service = ConversationService()
+
+    def validate_request(
+        self, data: Dict[str, Any], require_conversation_id: bool = False
+    ) -> Optional[Response]:
+        """Common request validation"""
+        required_fields = ["question"]
+        if require_conversation_id:
+            required_fields.append("conversation_id")
+        if missing_fields := check_required_fields(data, required_fields):
+            return missing_fields
+        return None
+
+    def complete_stream(
+        self,
+        question: str,
+        agent: Any,
+        retriever: Any,
+        conversation_id: Optional[str],
+        user_api_key: Optional[str],
+        decoded_token: Dict[str, Any],
+        isNoneDoc: bool = False,
+        index: Optional[int] = None,
+        should_save_conversation: bool = True,
+        attachment_ids: Optional[List[str]] = None,
+        agent_id: Optional[str] = None,
+        is_shared_usage: bool = False,
+        shared_token: Optional[str] = None,
+    ) -> Generator[str, None, None]:
+        """
+        Generator function that streams the complete conversation response.
+
+        Args:
+            question: The user's question
+            agent: The agent instance
+            retriever: The retriever instance
+            conversation_id: Existing conversation ID
+            user_api_key: User's API key if any
+            decoded_token: Decoded JWT token
+            isNoneDoc: Flag for document-less responses
+            index: Index of message to update
+            should_save_conversation: Whether to persist the conversation
+            attachment_ids: List of attachment IDs
+            agent_id: ID of agent used
+            is_shared_usage: Flag for shared agent usage
+            shared_token: Token for shared agent
+
+        Yields:
+            Server-sent event strings
+        """
+        try:
+            response_full, thought, source_log_docs, tool_calls = "", "", [], []
+            is_structured = False
+            schema_info = None
+            structured_chunks = []
+
+            for line in agent.gen(query=question, retriever=retriever):
+                if "answer" in line:
+                    response_full += str(line["answer"])
+                    if line.get("structured"):
+                        is_structured = True
+                        schema_info = line.get("schema")
+                        structured_chunks.append(line["answer"])
+                    else:
+                        data = json.dumps({"type": "answer", "answer": line["answer"]})
+                        yield f"data: {data}\n\n"
+                elif "sources" in line:
+                    truncated_sources = []
+                    source_log_docs = line["sources"]
+                    for source in line["sources"]:
+                        truncated_source = source.copy()
+                        if "text" in truncated_source:
+                            truncated_source["text"] = (
+                                truncated_source["text"][:100].strip() + "..."
+                            )
+                        truncated_sources.append(truncated_source)
+                    if truncated_sources:
+                        data = json.dumps(
+                            {"type": "source", "source": truncated_sources}
+                        )
+                        yield f"data: {data}\n\n"
+                elif "tool_calls" in line:
+                    tool_calls = line["tool_calls"]
+                elif "thought" in line:
+                    thought += line["thought"]
+                    data = json.dumps({"type": "thought", "thought": line["thought"]})
+                    yield f"data: {data}\n\n"
+                elif "type" in line:
+                    data = json.dumps(line)
+                    yield f"data: {data}\n\n"
+
+            if is_structured and structured_chunks:
+                structured_data = {
+                    "type": "structured_answer",
+                    "answer": response_full,
+                    "structured": True,
+                    "schema": schema_info,
+                }
+                data = json.dumps(structured_data)
+                yield f"data: {data}\n\n"
+
+            if isNoneDoc:
+                for doc in source_log_docs:
+                    doc["source"] = "None"
+            llm = LLMCreator.create_llm(
+                settings.LLM_PROVIDER,
+                api_key=settings.API_KEY,
+                user_api_key=user_api_key,
+                decoded_token=decoded_token,
+            )
+
+            if should_save_conversation:
+                conversation_id = self.conversation_service.save_conversation(
+                    conversation_id,
+                    question,
+                    response_full,
+                    thought,
+                    source_log_docs,
+                    tool_calls,
+                    llm,
+                    self.gpt_model,
+                    decoded_token,
+                    index=index,
+                    api_key=user_api_key,
+                    agent_id=agent_id,
+                    is_shared_usage=is_shared_usage,
+                    shared_token=shared_token,
+                    attachment_ids=attachment_ids,
+                )
+            else:
+                conversation_id = None
+            id_data = {"type": "id", "id": str(conversation_id)}
+            data = json.dumps(id_data)
+            yield f"data: {data}\n\n"
+
+            retriever_params = retriever.get_params()
+            log_data = {
+                "action": "stream_answer",
+                "level": "info",
+                "user": decoded_token.get("sub"),
+                "api_key": user_api_key,
+                "question": question,
+                "response": response_full,
+                "sources": source_log_docs,
+                "retriever_params": retriever_params,
+                "attachments": attachment_ids,
+                "timestamp": datetime.datetime.now(datetime.timezone.utc),
+            }
+            if is_structured:
+                log_data["structured_output"] = True
+                if schema_info:
+                    log_data["schema"] = schema_info
+  
+            # clean up text fields to be no longer than 10000 characters
+            for key, value in log_data.items():
+                if isinstance(value, str) and len(value) > 10000:
+                    log_data[key] = value[:10000]
+            
+            self.user_logs_collection.insert_one(log_data)
+
+            # End of stream
+
+            data = json.dumps({"type": "end"})
+            yield f"data: {data}\n\n"
+        except Exception as e:
+            logger.error(f"Error in stream: {str(e)}", exc_info=True)
+            data = json.dumps(
+                {
+                    "type": "error",
+                    "error": "Please try again later. We apologize for any inconvenience.",
+                }
+            )
+            yield f"data: {data}\n\n"
+            return
+
+    def process_response_stream(self, stream):
+        """Process the stream response for non-streaming endpoint"""
+        conversation_id = ""
+        response_full = ""
+        source_log_docs = []
+        tool_calls = []
+        thought = ""
+        stream_ended = False
+        is_structured = False
+        schema_info = None
+
+        for line in stream:
+            try:
+                event_data = line.replace("data: ", "").strip()
+                event = json.loads(event_data)
+
+                if event["type"] == "id":
+                    conversation_id = event["id"]
+                elif event["type"] == "answer":
+                    response_full += event["answer"]
+                elif event["type"] == "structured_answer":
+                    response_full = event["answer"]
+                    is_structured = True
+                    schema_info = event.get("schema")
+                elif event["type"] == "source":
+                    source_log_docs = event["source"]
+                elif event["type"] == "tool_calls":
+                    tool_calls = event["tool_calls"]
+                elif event["type"] == "thought":
+                    thought = event["thought"]
+                elif event["type"] == "error":
+                    logger.error(f"Error from stream: {event['error']}")
+                    return None, None, None, None, event["error"]
+                elif event["type"] == "end":
+                    stream_ended = True
+            except (json.JSONDecodeError, KeyError) as e:
+                logger.warning(f"Error parsing stream event: {e}, line: {line}")
+                continue
+        if not stream_ended:
+            logger.error("Stream ended unexpectedly without an 'end' event.")
+            return None, None, None, None, "Stream ended unexpectedly"
+
+        result = (
+            conversation_id,
+            response_full,
+            source_log_docs,
+            tool_calls,
+            thought,
+            None,
+        )
+
+        if is_structured:
+            result = result + ({"structured": True, "schema": schema_info},)
+
+        return result
+
+    def error_stream_generate(self, err_response):
+        data = json.dumps({"type": "error", "error": err_response})
+        yield f"data: {data}\n\n"

application/api/answer/routes/stream.py

@@ -0,0 +1,117 @@
+import logging
+import traceback
+
+from flask import request, Response
+from flask_restx import fields, Resource
+
+from application.api import api
+
+from application.api.answer.routes.base import answer_ns, BaseAnswerResource
+
+from application.api.answer.services.stream_processor import StreamProcessor
+
+logger = logging.getLogger(__name__)
+
+
+@answer_ns.route("/stream")
+class StreamResource(Resource, BaseAnswerResource):
+    def __init__(self, *args, **kwargs):
+        Resource.__init__(self, *args, **kwargs)
+        BaseAnswerResource.__init__(self)
+
+    stream_model = answer_ns.model(
+        "StreamModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
+            ),
+            "history": fields.List(
+                fields.String,
+                required=False,
+                description="Conversation history (only for new conversations)",
+            ),
+            "conversation_id": fields.String(
+                required=False,
+                description="Existing conversation ID (loads history)",
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+            "index": fields.Integer(
+                required=False, description="Index of the query to update"
+            ),
+            "save_conversation": fields.Boolean(
+                required=False,
+                default=True,
+                description="Whether to save the conversation",
+            ),
+            "attachments": fields.List(
+                fields.String, required=False, description="List of attachment IDs"
+            ),
+        },
+    )
+
+    @api.expect(stream_model)
+    @api.doc(description="Stream a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        if error := self.validate_request(data, "index" in data):
+            return error
+        decoded_token = getattr(request, "decoded_token", None)
+        processor = StreamProcessor(data, decoded_token)
+        try:
+            processor.initialize()
+            agent = processor.create_agent()
+            retriever = processor.create_retriever()
+
+            return Response(
+                self.complete_stream(
+                    question=data["question"],
+                    agent=agent,
+                    retriever=retriever,
+                    conversation_id=processor.conversation_id,
+                    user_api_key=processor.agent_config.get("user_api_key"),
+                    decoded_token=processor.decoded_token,
+                    isNoneDoc=data.get("isNoneDoc"),
+                    index=data.get("index"),
+                    should_save_conversation=data.get("save_conversation", True),
+                    attachment_ids=data.get("attachments", []),
+                    agent_id=data.get("agent_id"),
+                    is_shared_usage=processor.is_shared_usage,
+                    shared_token=processor.shared_token,
+                ),
+                mimetype="text/event-stream",
+            )
+        except ValueError as e:
+            message = "Malformed request body"
+            logger.error(
+                f"/stream - error: {message} - specific error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return Response(
+                self.error_stream_generate(message),
+                status=400,
+                mimetype="text/event-stream",
+            )
+        except Exception as e:
+            logger.error(
+                f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return Response(
+                self.error_stream_generate("Unknown error occurred"),
+                status=400,
+                mimetype="text/event-stream",
+            )

application/api/answer/services/conversation_service.py

@@ -0,0 +1,180 @@
+import logging
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from application.core.mongo_db import MongoDB
+
+from application.core.settings import settings
+from bson import ObjectId
+
+
+logger = logging.getLogger(__name__)
+
+
+class ConversationService:
+    def __init__(self):
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        self.conversations_collection = db["conversations"]
+        self.agents_collection = db["agents"]
+
+    def get_conversation(
+        self, conversation_id: str, user_id: str
+    ) -> Optional[Dict[str, Any]]:
+        """Retrieve a conversation with proper access control"""
+        if not conversation_id or not user_id:
+            return None
+        try:
+            conversation = self.conversations_collection.find_one(
+                {
+                    "_id": ObjectId(conversation_id),
+                    "$or": [{"user": user_id}, {"shared_with": user_id}],
+                }
+            )
+
+            if not conversation:
+                logger.warning(
+                    f"Conversation not found or unauthorized - ID: {conversation_id}, User: {user_id}"
+                )
+                return None
+            conversation["_id"] = str(conversation["_id"])
+            return conversation
+        except Exception as e:
+            logger.error(f"Error fetching conversation: {str(e)}", exc_info=True)
+            return None
+
+    def save_conversation(
+        self,
+        conversation_id: Optional[str],
+        question: str,
+        response: str,
+        thought: str,
+        sources: List[Dict[str, Any]],
+        tool_calls: List[Dict[str, Any]],
+        llm: Any,
+        gpt_model: str,
+        decoded_token: Dict[str, Any],
+        index: Optional[int] = None,
+        api_key: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        is_shared_usage: bool = False,
+        shared_token: Optional[str] = None,
+        attachment_ids: Optional[List[str]] = None,
+    ) -> str:
+        """Save or update a conversation in the database"""
+        user_id = decoded_token.get("sub")
+        if not user_id:
+            raise ValueError("User ID not found in token")
+        current_time = datetime.now(timezone.utc)
+        
+        # clean up in sources array such that we save max 1k characters for text part
+        for source in sources:
+            if "text" in source and isinstance(source["text"], str):
+                source["text"] = source["text"][:1000]
+
+        if conversation_id is not None and index is not None:
+            # Update existing conversation with new query
+
+            result = self.conversations_collection.update_one(
+                {
+                    "_id": ObjectId(conversation_id),
+                    "user": user_id,
+                    f"queries.{index}": {"$exists": True},
+                },
+                {
+                    "$set": {
+                        f"queries.{index}.prompt": question,
+                        f"queries.{index}.response": response,
+                        f"queries.{index}.thought": thought,
+                        f"queries.{index}.sources": sources,
+                        f"queries.{index}.tool_calls": tool_calls,
+                        f"queries.{index}.timestamp": current_time,
+                        f"queries.{index}.attachments": attachment_ids,
+                    }
+                },
+            )
+
+            if result.matched_count == 0:
+                raise ValueError("Conversation not found or unauthorized")
+            self.conversations_collection.update_one(
+                {
+                    "_id": ObjectId(conversation_id),
+                    "user": user_id,
+                    f"queries.{index}": {"$exists": True},
+                },
+                {"$push": {"queries": {"$each": [], "$slice": index + 1}}},
+            )
+            return conversation_id
+        elif conversation_id:
+            # Append new message to existing conversation
+
+            result = self.conversations_collection.update_one(
+                {"_id": ObjectId(conversation_id), "user": user_id},
+                {
+                    "$push": {
+                        "queries": {
+                            "prompt": question,
+                            "response": response,
+                            "thought": thought,
+                            "sources": sources,
+                            "tool_calls": tool_calls,
+                            "timestamp": current_time,
+                            "attachments": attachment_ids,
+                        }
+                    }
+                },
+            )
+
+            if result.matched_count == 0:
+                raise ValueError("Conversation not found or unauthorized")
+            return conversation_id
+        else:
+            # Create new conversation
+
+            messages_summary = [
+                {
+                    "role": "assistant",
+                    "content": "Summarise following conversation in no more than 3 "
+                    "words, respond ONLY with the summary, use the same "
+                    "language as the user query",
+                },
+                {
+                    "role": "user",
+                    "content": "Summarise following conversation in no more than 3 words, "
+                    "respond ONLY with the summary, use the same language as the "
+                    "user query \n\nUser: " + question + "\n\n" + "AI: " + response,
+                },
+            ]
+
+            completion = llm.gen(
+                model=gpt_model, messages=messages_summary, max_tokens=30
+            )
+
+            conversation_data = {
+                "user": user_id,
+                "date": current_time,
+                "name": completion,
+                "queries": [
+                    {
+                        "prompt": question,
+                        "response": response,
+                        "thought": thought,
+                        "sources": sources,
+                        "tool_calls": tool_calls,
+                        "timestamp": current_time,
+                        "attachments": attachment_ids,
+                    }
+                ],
+            }
+
+            if api_key:
+                if agent_id:
+                    conversation_data["agent_id"] = agent_id
+                    if is_shared_usage:
+                        conversation_data["is_shared_usage"] = is_shared_usage
+                        conversation_data["shared_token"] = shared_token
+                agent = self.agents_collection.find_one({"key": api_key})
+                if agent:
+                    conversation_data["api_key"] = agent["key"]
+            result = self.conversations_collection.insert_one(conversation_data)
+            return str(result.inserted_id)

application/api/answer/services/stream_processor.py

@@ -0,0 +1,277 @@
+import datetime
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from bson.dbref import DBRef
+
+from bson.objectid import ObjectId
+
+from application.agents.agent_creator import AgentCreator
+from application.api.answer.services.conversation_service import ConversationService
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.retriever.retriever_creator import RetrieverCreator
+from application.utils import get_gpt_model, limit_chat_history
+
+logger = logging.getLogger(__name__)
+
+
+def get_prompt(prompt_id: str, prompts_collection=None) -> str:
+    """
+    Get a prompt by preset name or MongoDB ID
+    """
+    current_dir = Path(__file__).resolve().parents[3]
+    prompts_dir = current_dir / "prompts"
+
+    preset_mapping = {
+        "default": "chat_combine_default.txt",
+        "creative": "chat_combine_creative.txt",
+        "strict": "chat_combine_strict.txt",
+        "reduce": "chat_reduce_prompt.txt",
+    }
+
+    if prompt_id in preset_mapping:
+        file_path = os.path.join(prompts_dir, preset_mapping[prompt_id])
+        try:
+            with open(file_path, "r") as f:
+                return f.read()
+        except FileNotFoundError:
+            raise FileNotFoundError(f"Prompt file not found: {file_path}")
+    try:
+        if prompts_collection is None:
+            mongo = MongoDB.get_client()
+            db = mongo[settings.MONGO_DB_NAME]
+            prompts_collection = db["prompts"]
+        prompt_doc = prompts_collection.find_one({"_id": ObjectId(prompt_id)})
+        if not prompt_doc:
+            raise ValueError(f"Prompt with ID {prompt_id} not found")
+        return prompt_doc["content"]
+    except Exception as e:
+        raise ValueError(f"Invalid prompt ID: {prompt_id}") from e
+
+
+class StreamProcessor:
+    def __init__(
+        self, request_data: Dict[str, Any], decoded_token: Optional[Dict[str, Any]]
+    ):
+        mongo = MongoDB.get_client()
+        self.db = mongo[settings.MONGO_DB_NAME]
+        self.agents_collection = self.db["agents"]
+        self.attachments_collection = self.db["attachments"]
+        self.prompts_collection = self.db["prompts"]
+
+        self.data = request_data
+        self.decoded_token = decoded_token
+        self.initial_user_id = (
+            self.decoded_token.get("sub") if self.decoded_token is not None else None
+        )
+        self.conversation_id = self.data.get("conversation_id")
+        self.source = (
+            {"active_docs": self.data["active_docs"]}
+            if "active_docs" in self.data
+            else {}
+        )
+        self.attachments = []
+        self.history = []
+        self.agent_config = {}
+        self.retriever_config = {}
+        self.is_shared_usage = False
+        self.shared_token = None
+        self.gpt_model = get_gpt_model()
+        self.conversation_service = ConversationService()
+
+    def initialize(self):
+        """Initialize all required components for processing"""
+        self._configure_retriever()
+        self._configure_agent()
+        self._load_conversation_history()
+        self._process_attachments()
+
+    def _load_conversation_history(self):
+        """Load conversation history either from DB or request"""
+        if self.conversation_id and self.initial_user_id:
+            conversation = self.conversation_service.get_conversation(
+                self.conversation_id, self.initial_user_id
+            )
+            if not conversation:
+                raise ValueError("Conversation not found or unauthorized")
+            self.history = [
+                {"prompt": query["prompt"], "response": query["response"]}
+                for query in conversation.get("queries", [])
+            ]
+        else:
+            self.history = limit_chat_history(
+                json.loads(self.data.get("history", "[]")), gpt_model=self.gpt_model
+            )
+
+    def _process_attachments(self):
+        """Process any attachments in the request"""
+        attachment_ids = self.data.get("attachments", [])
+        self.attachments = self._get_attachments_content(
+            attachment_ids, self.initial_user_id
+        )
+
+    def _get_attachments_content(self, attachment_ids, user_id):
+        """
+        Retrieve content from attachment documents based on their IDs.
+        """
+        if not attachment_ids:
+            return []
+        attachments = []
+        for attachment_id in attachment_ids:
+            try:
+                attachment_doc = self.attachments_collection.find_one(
+                    {"_id": ObjectId(attachment_id), "user": user_id}
+                )
+
+                if attachment_doc:
+                    attachments.append(attachment_doc)
+            except Exception as e:
+                logger.error(
+                    f"Error retrieving attachment {attachment_id}: {e}", exc_info=True
+                )
+        return attachments
+
+    def _get_agent_key(self, agent_id: Optional[str], user_id: Optional[str]) -> tuple:
+        """Get API key for agent with access control"""
+        if not agent_id:
+            return None, False, None
+        try:
+            agent = self.agents_collection.find_one({"_id": ObjectId(agent_id)})
+            if agent is None:
+                raise Exception("Agent not found")
+            is_owner = agent.get("user") == user_id
+            is_shared_with_user = agent.get(
+                "shared_publicly", False
+            ) or user_id in agent.get("shared_with", [])
+
+            if not (is_owner or is_shared_with_user):
+                raise Exception("Unauthorized access to the agent")
+            if is_owner:
+                self.agents_collection.update_one(
+                    {"_id": ObjectId(agent_id)},
+                    {
+                        "$set": {
+                            "lastUsedAt": datetime.datetime.now(datetime.timezone.utc)
+                        }
+                    },
+                )
+            return str(agent["key"]), not is_owner, agent.get("shared_token")
+        except Exception as e:
+            logger.error(f"Error in get_agent_key: {str(e)}", exc_info=True)
+            raise
+
+    def _get_data_from_api_key(self, api_key: str) -> Dict[str, Any]:
+        data = self.agents_collection.find_one({"key": api_key})
+        if not data:
+            raise Exception("Invalid API Key, please generate a new key", 401)
+        source = data.get("source")
+        if isinstance(source, DBRef):
+            source_doc = self.db.dereference(source)
+            data["source"] = str(source_doc["_id"])
+            data["retriever"] = source_doc.get("retriever", data.get("retriever"))
+            data["chunks"] = source_doc.get("chunks", data.get("chunks"))
+        else:
+            data["source"] = None
+        return data
+
+    def _configure_agent(self):
+        """Configure the agent based on request data"""
+        agent_id = self.data.get("agent_id")
+        self.agent_key, self.is_shared_usage, self.shared_token = self._get_agent_key(
+            agent_id, self.initial_user_id
+        )
+
+        api_key = self.data.get("api_key")
+        if api_key:
+            data_key = self._get_data_from_api_key(api_key)
+            self.agent_config.update(
+                {
+                    "prompt_id": data_key.get("prompt_id", "default"),
+                    "agent_type": data_key.get("agent_type", settings.AGENT_NAME),
+                    "user_api_key": api_key,
+                    "json_schema": data_key.get("json_schema"),
+                }
+            )
+            self.initial_user_id = data_key.get("user")
+            self.decoded_token = {"sub": data_key.get("user")}
+            if data_key.get("source"):
+                self.source = {"active_docs": data_key["source"]}
+            if data_key.get("retriever"):
+                self.retriever_config["retriever_name"] = data_key["retriever"]
+            if data_key.get("chunks") is not None:
+                self.retriever_config["chunks"] = data_key["chunks"]
+        elif self.agent_key:
+            data_key = self._get_data_from_api_key(self.agent_key)
+            self.agent_config.update(
+                {
+                    "prompt_id": data_key.get("prompt_id", "default"),
+                    "agent_type": data_key.get("agent_type", settings.AGENT_NAME),
+                    "user_api_key": self.agent_key,
+                    "json_schema": data_key.get("json_schema"),
+                }
+            )
+            self.decoded_token = (
+                self.decoded_token
+                if self.is_shared_usage
+                else {"sub": data_key.get("user")}
+            )
+            if data_key.get("source"):
+                self.source = {"active_docs": data_key["source"]}
+            if data_key.get("retriever"):
+                self.retriever_config["retriever_name"] = data_key["retriever"]
+            if data_key.get("chunks") is not None:
+                self.retriever_config["chunks"] = data_key["chunks"]
+        else:
+            self.agent_config.update(
+                {
+                    "prompt_id": self.data.get("prompt_id", "default"),
+                    "agent_type": settings.AGENT_NAME,
+                    "user_api_key": None,
+                    "json_schema": None,
+                }
+            )
+
+    def _configure_retriever(self):
+        """Configure the retriever based on request data"""
+        self.retriever_config = {
+            "retriever_name": self.data.get("retriever", "classic"),
+            "chunks": int(self.data.get("chunks", 2)),
+            "token_limit": self.data.get("token_limit", settings.DEFAULT_MAX_HISTORY),
+        }
+
+        if "isNoneDoc" in self.data and self.data["isNoneDoc"]:
+            self.retriever_config["chunks"] = 0
+
+    def create_agent(self):
+        """Create and return the configured agent"""
+        return AgentCreator.create_agent(
+            self.agent_config["agent_type"],
+            endpoint="stream",
+            llm_name=settings.LLM_PROVIDER,
+            gpt_model=self.gpt_model,
+            api_key=settings.API_KEY,
+            user_api_key=self.agent_config["user_api_key"],
+            prompt=get_prompt(self.agent_config["prompt_id"], self.prompts_collection),
+            chat_history=self.history,
+            decoded_token=self.decoded_token,
+            attachments=self.attachments,
+            json_schema=self.agent_config.get("json_schema"),
+        )
+
+    def create_retriever(self):
+        """Create and return the configured retriever"""
+        return RetrieverCreator.create_retriever(
+            self.retriever_config["retriever_name"],
+            source=self.source,
+            chat_history=self.history,
+            prompt=get_prompt(self.agent_config["prompt_id"], self.prompts_collection),
+            chunks=self.retriever_config["chunks"],
+            token_limit=self.retriever_config["token_limit"],
+            gpt_model=self.gpt_model,
+            user_api_key=self.agent_config["user_api_key"],
+            decoded_token=self.decoded_token,
+        )

application/api/connector/routes.py

@@ -0,0 +1,627 @@
+import datetime
+import json
+import logging
+
+
+from bson.objectid import ObjectId
+from flask import (
+    Blueprint,
+    current_app,
+    jsonify,
+    make_response,
+    request
+)
+from flask_restx import fields, Namespace, Resource
+
+
+
+
+from application.api.user.tasks import (
+    ingest_connector_task,
+)
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.api import api
+
+
+from application.utils import (
+    check_required_fields
+)
+
+
+from application.parser.connectors.connector_creator import ConnectorCreator
+
+
+
+mongo = MongoDB.get_client()
+db = mongo[settings.MONGO_DB_NAME]
+sources_collection = db["sources"]
+sessions_collection = db["connector_sessions"]
+
+connector = Blueprint("connector", __name__)
+connectors_ns = Namespace("connectors", description="Connector operations", path="/")
+api.add_namespace(connectors_ns)
+
+
+
+@connectors_ns.route("/api/connectors/upload")
+class UploadConnector(Resource):
+    @api.expect(
+        api.model(
+            "ConnectorUploadModel",
+            {
+                "user": fields.String(required=True, description="User ID"),
+                "source": fields.String(
+                    required=True, description="Source type (google_drive, github, etc.)"
+                ),
+                "name": fields.String(required=True, description="Job name"),
+                "data": fields.String(required=True, description="Configuration data"),
+                "repo_url": fields.String(description="GitHub repository URL"),
+            },
+        )
+    )
+    @api.doc(
+        description="Uploads connector source for vectorization",
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        data = request.form
+        required_fields = ["user", "source", "name", "data"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            config = json.loads(data["data"])
+            source_data = None
+            sync_frequency = config.get("sync_frequency", "never")
+
+            if data["source"] == "github":
+                source_data = config.get("repo_url")
+            elif data["source"] in ["crawler", "url"]:
+                source_data = config.get("url")
+            elif data["source"] == "reddit":
+                source_data = config
+            elif data["source"] in ConnectorCreator.get_supported_connectors():
+                session_token = config.get("session_token")
+                if not session_token:
+                    return make_response(jsonify({
+                        "success": False,
+                        "error": f"Missing session_token in {data['source']} configuration"
+                    }), 400)
+
+                file_ids = config.get("file_ids", [])
+                if isinstance(file_ids, str):
+                    file_ids = [id.strip() for id in file_ids.split(',') if id.strip()]
+                elif not isinstance(file_ids, list):
+                    file_ids = []
+
+                folder_ids = config.get("folder_ids", [])
+                if isinstance(folder_ids, str):
+                    folder_ids = [id.strip() for id in folder_ids.split(',') if id.strip()]
+                elif not isinstance(folder_ids, list):
+                    folder_ids = []
+
+                config["file_ids"] = file_ids
+                config["folder_ids"] = folder_ids
+
+                task = ingest_connector_task.delay(
+                    job_name=data["name"],
+                    user=decoded_token.get("sub"),
+                    source_type=data["source"],
+                    session_token=session_token,
+                    file_ids=file_ids,
+                    folder_ids=folder_ids,
+                    recursive=config.get("recursive", False),
+                    retriever=config.get("retriever", "classic"),
+                    sync_frequency=sync_frequency
+                )
+                return make_response(jsonify({"success": True, "task_id": task.id}), 200)
+            task = ingest_connector_task.delay(
+                source_data=source_data,
+                job_name=data["name"],
+                user=decoded_token.get("sub"),
+                loader=data["source"],
+                sync_frequency=sync_frequency
+            )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error uploading connector source: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True, "task_id": task.id}), 200)
+
+
+@connectors_ns.route("/api/connectors/task_status")
+class ConnectorTaskStatus(Resource):
+    task_status_model = api.model(
+        "ConnectorTaskStatusModel",
+        {"task_id": fields.String(required=True, description="Task ID")},
+    )
+
+    @api.expect(task_status_model)
+    @api.doc(description="Get connector task status")
+    def get(self):
+        task_id = request.args.get("task_id")
+        if not task_id:
+            return make_response(
+                jsonify({"success": False, "message": "Task ID is required"}), 400
+            )
+        try:
+            from application.celery_init import celery
+
+            task = celery.AsyncResult(task_id)
+            task_meta = task.info
+            print(f"Task status: {task.status}")
+            if not isinstance(
+                task_meta, (dict, list, str, int, float, bool, type(None))
+            ):
+                task_meta = str(task_meta)
+        except Exception as err:
+            current_app.logger.error(f"Error getting task status: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"status": task.status, "result": task_meta}), 200)
+
+
+@connectors_ns.route("/api/connectors/sources")
+class ConnectorSources(Resource):
+    @api.doc(description="Get connector sources")
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            sources = sources_collection.find({"user": user, "type": "connector"}).sort("date", -1)
+            connector_sources = []
+            for source in sources:
+                connector_sources.append({
+                    "id": str(source["_id"]),
+                    "name": source.get("name"),
+                    "date": source.get("date"),
+                    "type": source.get("type"),
+                    "source": source.get("source"),
+                    "tokens": source.get("tokens", ""),
+                    "retriever": source.get("retriever", "classic"),
+                    "syncFrequency": source.get("sync_frequency", ""),
+                })
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving connector sources: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify(connector_sources), 200)
+
+
+@connectors_ns.route("/api/connectors/delete")
+class DeleteConnectorSource(Resource):
+    @api.doc(
+        description="Delete a connector source",
+        params={"source_id": "The source ID to delete"},
+    )
+    def delete(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        source_id = request.args.get("source_id")
+        if not source_id:
+            return make_response(
+                jsonify({"success": False, "message": "source_id is required"}), 400
+            )
+        try:
+            result = sources_collection.delete_one(
+                {"_id": ObjectId(source_id), "user": decoded_token.get("sub")}
+            )
+            if result.deleted_count == 0:
+                return make_response(
+                    jsonify({"success": False, "message": "Source not found"}), 404
+                )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error deleting connector source: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@connectors_ns.route("/api/connectors/auth")
+class ConnectorAuth(Resource):
+    @api.doc(description="Get connector OAuth authorization URL", params={"provider": "Connector provider (e.g., google_drive)"})
+    def get(self):
+        try:
+            provider = request.args.get('provider') or request.args.get('source')
+            if not provider:
+                return make_response(jsonify({"success": False, "error": "Missing provider"}), 400)
+
+            if not ConnectorCreator.is_supported(provider):
+                return make_response(jsonify({"success": False, "error": f"Unsupported provider: {provider}"}), 400)
+
+            import uuid
+            state = str(uuid.uuid4())
+            auth = ConnectorCreator.create_auth(provider)
+            authorization_url = auth.get_authorization_url(state=state)
+            return make_response(jsonify({
+                "success": True,
+                "authorization_url": authorization_url,
+                "state": state
+            }), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error generating connector auth URL: {e}")
+            return make_response(jsonify({"success": False, "error": str(e)}), 500)
+
+
+@connectors_ns.route("/api/connectors/callback")
+class ConnectorsCallback(Resource):
+    @api.doc(description="Handle OAuth callback for external connectors")
+    def get(self):
+        """Handle OAuth callback for external connectors"""
+        try:
+            from application.parser.connectors.connector_creator import ConnectorCreator
+            from flask import request, redirect
+            import uuid
+
+            provider = request.args.get('provider', 'google_drive')
+            authorization_code = request.args.get('code')
+            _ = request.args.get('state')
+            error = request.args.get('error')
+
+            if error:
+                return redirect(f"/api/connectors/callback-status?status=error&message=OAuth+error:+{error}.+Please+try+again+and+make+sure+to+grant+all+requested+permissions,+including+offline+access.&provider={provider}")
+
+            if not authorization_code:
+                return redirect(f"/api/connectors/callback-status?status=error&message=Authorization+code+not+provided.+Please+complete+the+authorization+process+and+make+sure+to+grant+offline+access.&provider={provider}")
+
+            try:
+                auth = ConnectorCreator.create_auth(provider)
+                token_info = auth.exchange_code_for_tokens(authorization_code)
+
+                session_token = str(uuid.uuid4())
+                
+
+                try:
+                    credentials = auth.create_credentials_from_token_info(token_info)
+                    service = auth.build_drive_service(credentials)
+                    user_info = service.about().get(fields="user").execute()
+                    user_email = user_info.get('user', {}).get('emailAddress', 'Connected User')
+                except Exception as e:
+                    current_app.logger.warning(f"Could not get user info: {e}")
+                    user_email = 'Connected User'
+
+                sanitized_token_info = {
+                    "access_token": token_info.get("access_token"),
+                    "refresh_token": token_info.get("refresh_token"),
+                    "token_uri": token_info.get("token_uri"),
+                    "expiry": token_info.get("expiry"),
+                    "scopes": token_info.get("scopes")
+                }
+
+                user_id = request.decoded_token.get("sub") if getattr(request, "decoded_token", None) else None
+                sessions_collection.insert_one({
+                    "session_token": session_token,
+                    "user": user_id,
+                    "token_info": sanitized_token_info,
+                    "created_at": datetime.datetime.now(datetime.timezone.utc),
+                    "user_email": user_email,
+                    "provider": provider
+                })
+
+                # Redirect to success page with session token and user email
+                return redirect(f"/api/connectors/callback-status?status=success&message=Authentication+successful&provider={provider}&session_token={session_token}&user_email={user_email}")
+
+            except Exception as e:
+                current_app.logger.error(f"Error exchanging code for tokens: {str(e)}", exc_info=True)
+                return redirect(f"/api/connectors/callback-status?status=error&message=Failed+to+exchange+authorization+code+for+tokens:+{str(e)}&provider={provider}")
+
+        except Exception as e:
+            current_app.logger.error(f"Error handling connector callback: {e}")
+            return redirect(f"/api/connectors/callback-status?status=error&message=Failed+to+complete+connector+authentication:+{str(e)}.+Please+try+again+and+make+sure+to+grant+all+requested+permissions,+including+offline+access.")
+
+
+@connectors_ns.route("/api/connectors/refresh")
+class ConnectorRefresh(Resource):
+    @api.expect(api.model("ConnectorRefreshModel", {"provider": fields.String(required=True), "refresh_token": fields.String(required=True)}))
+    @api.doc(description="Refresh connector access token")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            refresh_token = data.get('refresh_token')
+
+            if not provider or not refresh_token:
+                return make_response(jsonify({"success": False, "error": "provider and refresh_token are required"}), 400)
+
+            auth = ConnectorCreator.create_auth(provider)
+            token_info = auth.refresh_access_token(refresh_token)
+            return make_response(jsonify({"success": True, "token_info": token_info}), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error refreshing token for connector: {e}")
+            return make_response(jsonify({"success": False, "error": str(e)}), 500)
+
+
+@connectors_ns.route("/api/connectors/files")
+class ConnectorFiles(Resource):
+    @api.expect(api.model("ConnectorFilesModel", {"provider": fields.String(required=True), "session_token": fields.String(required=True), "folder_id": fields.String(required=False), "limit": fields.Integer(required=False), "page_token": fields.String(required=False)}))
+    @api.doc(description="List files from a connector provider (supports pagination)")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            session_token = data.get('session_token')
+            folder_id = data.get('folder_id')
+            limit = data.get('limit', 10)
+            page_token = data.get('page_token')
+            if not provider or not session_token:
+                return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400)
+
+
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+            user = decoded_token.get('sub')
+            session = sessions_collection.find_one({"session_token": session_token, "user": user})
+            if not session:
+                return make_response(jsonify({"success": False, "error": "Invalid or unauthorized session"}), 401)
+
+            loader = ConnectorCreator.create_connector(provider, session_token)
+            documents = loader.load_data({
+                'limit': limit,
+                'list_only': True,
+                'session_token': session_token,
+                'folder_id': folder_id,
+                'page_token': page_token
+            })
+
+            files = []
+            for doc in documents[:limit]:
+                metadata = doc.extra_info
+                modified_time = metadata.get('modified_time')
+                if modified_time:
+                    date_part = modified_time.split('T')[0]
+                    time_part = modified_time.split('T')[1].split('.')[0].split('Z')[0]
+                    formatted_time = f"{date_part} {time_part}"
+                else:
+                    formatted_time = None
+
+                files.append({
+                    'id': doc.doc_id,
+                    'name': metadata.get('file_name', 'Unknown File'),
+                    'type': metadata.get('mime_type', 'unknown'),
+                    'size': metadata.get('size', None),
+                    'modifiedTime': formatted_time
+                })
+
+            next_token = getattr(loader, 'next_page_token', None)
+            has_more = bool(next_token)
+
+            return make_response(jsonify({"success": True, "files": files, "total": len(files), "next_page_token": next_token, "has_more": has_more}), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error loading connector files: {e}")
+            return make_response(jsonify({"success": False, "error": f"Failed to load files: {str(e)}"}), 500)
+
+
+@connectors_ns.route("/api/connectors/validate-session")
+class ConnectorValidateSession(Resource):
+    @api.expect(api.model("ConnectorValidateSessionModel", {"provider": fields.String(required=True), "session_token": fields.String(required=True)}))
+    @api.doc(description="Validate connector session token and return user info")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            session_token = data.get('session_token')
+            if not provider or not session_token:
+                return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400)
+
+
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+            user = decoded_token.get('sub')
+
+            session = sessions_collection.find_one({"session_token": session_token, "user": user})
+            if not session or "token_info" not in session:
+                return make_response(jsonify({"success": False, "error": "Invalid or expired session"}), 401)
+
+            token_info = session["token_info"]
+            auth = ConnectorCreator.create_auth(provider)
+            is_expired = auth.is_token_expired(token_info)
+
+            return make_response(jsonify({
+                "success": True,
+                "expired": is_expired,
+                "user_email": session.get('user_email', 'Connected User')
+            }), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error validating connector session: {e}")
+            return make_response(jsonify({"success": False, "error": str(e)}), 500)
+
+
+@connectors_ns.route("/api/connectors/disconnect")
+class ConnectorDisconnect(Resource):
+    @api.expect(api.model("ConnectorDisconnectModel", {"provider": fields.String(required=True), "session_token": fields.String(required=False)}))
+    @api.doc(description="Disconnect a connector session")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            session_token = data.get('session_token')
+            if not provider:
+                return make_response(jsonify({"success": False, "error": "provider is required"}), 400)
+
+
+            if session_token:
+                sessions_collection.delete_one({"session_token": session_token})
+            
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error disconnecting connector session: {e}")
+            return make_response(jsonify({"success": False, "error": str(e)}), 500)
+
+
+@connectors_ns.route("/api/connectors/sync")
+class ConnectorSync(Resource):
+    @api.expect(
+        api.model(
+            "ConnectorSyncModel",
+            {
+                "source_id": fields.String(required=True, description="Source ID to sync"),
+                "session_token": fields.String(required=True, description="Authentication token")
+            },
+        )
+    )
+    @api.doc(description="Sync connector source to check for modifications")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+
+        try:
+            data = request.get_json()
+            source_id = data.get('source_id')
+            session_token = data.get('session_token')
+
+            if not all([source_id, session_token]):
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "source_id and session_token are required"
+                    }), 
+                    400
+                )
+            source = sources_collection.find_one({"_id": ObjectId(source_id)})
+            if not source:
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "Source not found"
+                    }), 
+                    404
+                )
+
+            if source.get('user') != decoded_token.get('sub'):
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "Unauthorized access to source"
+                    }), 
+                    403
+                )
+
+            remote_data = {}
+            try:
+                if source.get('remote_data'):
+                    remote_data = json.loads(source.get('remote_data'))
+            except json.JSONDecodeError:
+                current_app.logger.error(f"Invalid remote_data format for source {source_id}")
+                remote_data = {}
+
+            source_type = remote_data.get('provider')
+            if not source_type:
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "Source provider not found in remote_data"
+                    }), 
+                    400
+                )
+
+            # Extract configuration from remote_data
+            file_ids = remote_data.get('file_ids', [])
+            folder_ids = remote_data.get('folder_ids', [])
+            recursive = remote_data.get('recursive', True)
+
+            # Start the sync task
+            task = ingest_connector_task.delay(
+                job_name=source.get('name'),
+                user=decoded_token.get('sub'),
+                source_type=source_type,
+                session_token=session_token,
+                file_ids=file_ids,
+                folder_ids=folder_ids,
+                recursive=recursive,
+                retriever=source.get('retriever', 'classic'),
+                operation_mode="sync",
+                doc_id=source_id,
+                sync_frequency=source.get('sync_frequency', 'never')
+            )
+
+            return make_response(
+                jsonify({
+                    "success": True,
+                    "task_id": task.id
+                }), 
+                200
+            )
+
+        except Exception as err:
+            current_app.logger.error(
+                f"Error syncing connector source: {err}",
+                exc_info=True
+            )
+            return make_response(
+                jsonify({
+                    "success": False,
+                    "error": str(err)
+                }), 
+                400
+            )
+
+
+@connectors_ns.route("/api/connectors/callback-status")
+class ConnectorCallbackStatus(Resource):
+    @api.doc(description="Return HTML page with connector authentication status")
+    def get(self):
+        """Return HTML page with connector authentication status"""
+        try:
+            status = request.args.get('status', 'error')
+            message = request.args.get('message', '')
+            provider = request.args.get('provider', 'connector')
+            session_token = request.args.get('session_token', '')
+            user_email = request.args.get('user_email', '')
+            
+            html_content = f"""
+            <!DOCTYPE html>
+            <html>
+            <head>
+                <title>{provider.replace('_', ' ').title()} Authentication</title>
+                <style>
+                    body {{ font-family: Arial, sans-serif; text-align: center; padding: 40px; }}
+                    .container {{ max-width: 600px; margin: 0 auto; }}
+                    .success {{ color: #4CAF50; }}
+                    .error {{ color: #F44336; }}
+                </style>
+                <script>
+                    window.onload = function() {{
+                        const status = "{status}";
+                        const sessionToken = "{session_token}";
+                        const userEmail = "{user_email}";
+                        
+                        if (status === "success" && window.opener) {{
+                            window.opener.postMessage({{
+                                type: '{provider}_auth_success',
+                                session_token: sessionToken,
+                                user_email: userEmail
+                            }}, '*');
+                            
+                            setTimeout(() => window.close(), 3000);
+                        }}
+                    }};
+                </script>
+            </head>
+            <body>
+                <div class="container">
+                    <h2>{provider.replace('_', ' ').title()} Authentication</h2>
+                    <div class="{status}">
+                        <p>{message}</p>
+                        {f'<p>Connected as: {user_email}</p>' if status == 'success' else ''}
+                    </div>
+                    <p><small>You can close this window. {f"Your {provider.replace('_', ' ').title()} is now connected and ready to use." if status == 'success' else ''}</small></p>
+                </div>
+            </body>
+            </html>
+            """
+            
+            return make_response(html_content, 200, {'Content-Type': 'text/html'})
+        except Exception as e:
+            current_app.logger.error(f"Error rendering callback status page: {e}")
+            return make_response("Authentication error occurred", 500, {'Content-Type': 'text/html'})
+
+

application/api/internal/routes.py

@@ -1,20 +1,29 @@
 import os
 import datetime
+import json
 from flask import Blueprint, request, send_from_directory
-from pymongo import MongoClient
 from werkzeug.utils import secure_filename
-
-
+from bson.objectid import ObjectId
+import logging
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
-mongo = MongoClient(settings.MONGO_URI)
-db = mongo["docsgpt"]
+from application.storage.storage_creator import StorageCreator
+
+
+logger = logging.getLogger(__name__)
+mongo = MongoDB.get_client()
+db = mongo[settings.MONGO_DB_NAME]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+
+internal = Blueprint("internal", __name__)
 
 
-internal = Blueprint('internal', __name__)
 @internal.route("/api/download", methods=["get"])
 def download_file():
     user = secure_filename(request.args.get("user"))
@@ -24,46 +33,95 @@ def download_file():
     return send_from_directory(save_dir, filename, as_attachment=True)
 
 
-
 @internal.route("/api/upload_index", methods=["POST"])
 def upload_index_files():
     """Upload two files(index.faiss, index.pkl) to the user's folder."""
     if "user" not in request.form:
         return {"status": "no user"}
-    user = secure_filename(request.form["user"])
+    user = request.form["user"] 
     if "name" not in request.form:
         return {"status": "no name"}
-    job_name = secure_filename(request.form["name"])
-    save_dir = os.path.join(current_dir, "indexes", user, job_name)
+    job_name = request.form["name"]
+    tokens = request.form["tokens"]
+    retriever = request.form["retriever"]
+    id = request.form["id"]
+    type = request.form["type"]
+    remote_data = request.form["remote_data"] if "remote_data" in request.form else None
+    sync_frequency = request.form["sync_frequency"] if "sync_frequency" in request.form else None
+    
+    file_path = request.form.get("file_path")
+    directory_structure = request.form.get("directory_structure")
+    
+    if directory_structure:
+        try:
+            directory_structure = json.loads(directory_structure)
+        except Exception:
+            logger.error("Error parsing directory_structure")
+            directory_structure = {}
+    else:
+        directory_structure = {}
+
+    storage = StorageCreator.get_storage()
+    index_base_path = f"indexes/{id}"
+    
     if settings.VECTOR_STORE == "faiss":
         if "file_faiss" not in request.files:
-            print("No file part")
+            logger.error("No file_faiss part")
             return {"status": "no file"}
         file_faiss = request.files["file_faiss"]
         if file_faiss.filename == "":
             return {"status": "no file name"}
         if "file_pkl" not in request.files:
-            print("No file part")
+            logger.error("No file_pkl part")
             return {"status": "no file"}
         file_pkl = request.files["file_pkl"]
         if file_pkl.filename == "":
             return {"status": "no file name"}
-        # saves index files
-        
-        if not os.path.exists(save_dir):
-            os.makedirs(save_dir)
-        file_faiss.save(os.path.join(save_dir, "index.faiss"))
-        file_pkl.save(os.path.join(save_dir, "index.pkl"))
-    # create entry in vectors_collection
-    vectors_collection.insert_one(
-        {
-            "user": user,
-            "name": job_name,
-            "language": job_name,
-            "location": save_dir,
-            "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
-            "model": settings.EMBEDDINGS_NAME,
-            "type": "local",
-        }
-    )
-    return {"status": "ok"}
+
+        # Save index files to storage
+        faiss_storage_path = f"{index_base_path}/index.faiss"
+        pkl_storage_path = f"{index_base_path}/index.pkl"
+        storage.save_file(file_faiss, faiss_storage_path)
+        storage.save_file(file_pkl, pkl_storage_path)
+
+
+    existing_entry = sources_collection.find_one({"_id": ObjectId(id)})
+    if existing_entry:
+        sources_collection.update_one(
+            {"_id": ObjectId(id)},
+            {
+                "$set": {
+                    "user": user,
+                    "name": job_name,
+                    "language": job_name,
+                    "date": datetime.datetime.now(),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "type": type,
+                    "tokens": tokens,
+                    "retriever": retriever,
+                    "remote_data": remote_data,
+                    "sync_frequency": sync_frequency,
+                    "file_path": file_path,
+                    "directory_structure": directory_structure,
+                }
+            },
+        )
+    else:
+        sources_collection.insert_one(
+            {
+                "_id": ObjectId(id),
+                "user": user,
+                "name": job_name,
+                "language": job_name,
+                "date": datetime.datetime.now(),
+                "model": settings.EMBEDDINGS_NAME,
+                "type": type,
+                "tokens": tokens,
+                "retriever": retriever,
+                "remote_data": remote_data,
+                "sync_frequency": sync_frequency,
+                "file_path": file_path,
+                "directory_structure": directory_structure,
+            }
+        )
+    return {"status": "ok"}

application/api/user/tasks.py

@@ -1,7 +1,96 @@
-from application.worker import ingest_worker
-from application.celery import celery
+from datetime import timedelta
+
+from application.celery_init import celery
+from application.worker import (
+    agent_webhook_worker,
+    attachment_worker,
+    ingest_worker,
+    remote_worker,
+    sync_worker,
+)
+
 
 @celery.task(bind=True)
-def ingest(self, directory, formats, name_job, filename, user):
-    resp = ingest_worker(self, directory, formats, name_job, filename, user)
+def ingest(self, directory, formats, job_name, user, file_path, filename):
+    resp = ingest_worker(self, directory, formats, job_name, file_path, filename, user)
     return resp
+
+
+@celery.task(bind=True)
+def ingest_remote(self, source_data, job_name, user, loader):
+    resp = remote_worker(self, source_data, job_name, user, loader)
+    return resp
+
+
+@celery.task(bind=True)
+def reingest_source_task(self, source_id, user):
+    from application.worker import reingest_source_worker
+    resp = reingest_source_worker(self, source_id, user)
+    return resp
+
+
+@celery.task(bind=True)
+def schedule_syncs(self, frequency):
+    resp = sync_worker(self, frequency)
+    return resp
+
+
+@celery.task(bind=True)
+def store_attachment(self, file_info, user):
+    resp = attachment_worker(self, file_info, user)
+    return resp
+
+
+@celery.task(bind=True)
+def process_agent_webhook(self, agent_id, payload):
+    resp = agent_webhook_worker(self, agent_id, payload)
+    return resp
+
+
+@celery.task(bind=True)
+def ingest_connector_task(
+    self,
+    job_name,
+    user,
+    source_type,
+    session_token=None,
+    file_ids=None,
+    folder_ids=None,
+    recursive=True,
+    retriever="classic",
+    operation_mode="upload",
+    doc_id=None,
+    sync_frequency="never"
+):
+    from application.worker import ingest_connector
+    resp = ingest_connector(
+        self,
+        job_name,
+        user,
+        source_type,
+        session_token=session_token,
+        file_ids=file_ids,
+        folder_ids=folder_ids,
+        recursive=recursive,
+        retriever=retriever,
+        operation_mode=operation_mode,
+        doc_id=doc_id,
+        sync_frequency=sync_frequency
+    )
+    return resp
+
+
+@celery.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+    sender.add_periodic_task(
+        timedelta(days=1),
+        schedule_syncs.s("daily"),
+    )
+    sender.add_periodic_task(
+        timedelta(weeks=1),
+        schedule_syncs.s("weekly"),
+    )
+    sender.add_periodic_task(
+        timedelta(days=30),
+        schedule_syncs.s("monthly"),
+    )

application/app.py

@@ -1,68 +1,115 @@
+import os
 import platform
-
+import uuid
 
 import dotenv
-from application.celery import celery
-from flask import Flask, request, redirect
+from flask import Flask, jsonify, redirect, request
+from jose import jwt
 
+from application.auth import handle_auth
 
-from application.core.settings import settings
-from application.api.user.routes import user
-from application.api.answer.routes import answer
-from application.api.internal.routes import internal
+from application.core.logging_config import setup_logging
 
+setup_logging()
 
+from application.api import api  # noqa: E402
+from application.api.answer import answer  # noqa: E402
+from application.api.internal.routes import internal  # noqa: E402
+from application.api.user.routes import user  # noqa: E402
+from application.api.connector.routes import connector  # noqa: E402
+from application.celery_init import celery  # noqa: E402
+from application.core.settings import settings  # noqa: E402
 
-# Redirect PosixPath to WindowsPath on Windows
 
 if platform.system() == "Windows":
     import pathlib
 
-    temp = pathlib.PosixPath
     pathlib.PosixPath = pathlib.WindowsPath
-
-# loading the .env file
 dotenv.load_dotenv()
 
-
-
 app = Flask(__name__)
 app.register_blueprint(user)
 app.register_blueprint(answer)
 app.register_blueprint(internal)
-app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER = "inputs"
-app.config["CELERY_BROKER_URL"] = settings.CELERY_BROKER_URL
-app.config["CELERY_RESULT_BACKEND"] = settings.CELERY_RESULT_BACKEND
-app.config["MONGO_URI"] = settings.MONGO_URI
+app.register_blueprint(connector)
+app.config.update(
+    UPLOAD_FOLDER="inputs",
+    CELERY_BROKER_URL=settings.CELERY_BROKER_URL,
+    CELERY_RESULT_BACKEND=settings.CELERY_RESULT_BACKEND,
+    MONGO_URI=settings.MONGO_URI,
+)
 celery.config_from_object("application.celeryconfig")
+api.init_app(app)
 
+if settings.AUTH_TYPE in ("simple_jwt", "session_jwt") and not settings.JWT_SECRET_KEY:
+    key_file = ".jwt_secret_key"
+    try:
+        with open(key_file, "r") as f:
+            settings.JWT_SECRET_KEY = f.read().strip()
+    except FileNotFoundError:
+        new_key = os.urandom(32).hex()
+        with open(key_file, "w") as f:
+            f.write(new_key)
+        settings.JWT_SECRET_KEY = new_key
+    except Exception as e:
+        raise RuntimeError(f"Failed to setup JWT_SECRET_KEY: {e}")
+SIMPLE_JWT_TOKEN = None
+if settings.AUTH_TYPE == "simple_jwt":
+    payload = {"sub": "local"}
+    SIMPLE_JWT_TOKEN = jwt.encode(payload, settings.JWT_SECRET_KEY, algorithm="HS256")
+    print(f"Generated Simple JWT Token: {SIMPLE_JWT_TOKEN}")
 
 
 @app.route("/")
 def home():
-    """
-    The frontend source code lives in the /frontend directory of the repository.
-    """
-    if request.remote_addr in ('0.0.0.0', '127.0.0.1', 'localhost', '172.18.0.1'):
-        # If users locally try to access DocsGPT running in Docker,
-        # they will be redirected to the Frontend application.
-        return redirect('http://localhost:5173')
+    if request.remote_addr in ("0.0.0.0", "127.0.0.1", "localhost", "172.18.0.1"):
+        return redirect("http://localhost:5173")
     else:
-        # Handle other cases or render the default page
-        return 'Welcome to DocsGPT Backend!'
+        return "Welcome to DocsGPT Backend!"
 
 
+@app.route("/api/config")
+def get_config():
+    response = {
+        "auth_type": settings.AUTH_TYPE,
+        "requires_auth": settings.AUTH_TYPE in ["simple_jwt", "session_jwt"],
+    }
+    return jsonify(response)
+
+
+@app.route("/api/generate_token")
+def generate_token():
+    if settings.AUTH_TYPE == "session_jwt":
+        new_user_id = str(uuid.uuid4())
+        token = jwt.encode(
+            {"sub": new_user_id}, settings.JWT_SECRET_KEY, algorithm="HS256"
+        )
+        return jsonify({"token": token})
+    return jsonify({"error": "Token generation not allowed in current auth mode"}), 400
+
+
+@app.before_request
+def authenticate_request():
+    if request.method == "OPTIONS":
+        return "", 200
+    decoded_token = handle_auth(request)
+    if not decoded_token:
+        request.decoded_token = None
+    elif "error" in decoded_token:
+        return jsonify(decoded_token), 401
+    else:
+        request.decoded_token = decoded_token
 
 
-# handling CORS
 @app.after_request
 def after_request(response):
     response.headers.add("Access-Control-Allow-Origin", "*")
-    response.headers.add("Access-Control-Allow-Headers", "Content-Type,Authorization")
-    response.headers.add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS")
-    # response.headers.add("Access-Control-Allow-Credentials", "true")
+    response.headers.add("Access-Control-Allow-Headers", "Content-Type, Authorization")
+    response.headers.add(
+        "Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS"
+    )
     return response
 
 
 if __name__ == "__main__":
-    app.run(debug=True, port=7091)
+    app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)

application/auth.py

@@ -0,0 +1,28 @@
+from jose import jwt
+
+from application.core.settings import settings
+
+
+def handle_auth(request, data={}):
+    if settings.AUTH_TYPE in ["simple_jwt", "session_jwt"]:
+        jwt_token = request.headers.get("Authorization")
+        if not jwt_token:
+            return None
+
+        jwt_token = jwt_token.replace("Bearer ", "")
+
+        try:
+            decoded_token = jwt.decode(
+                jwt_token,
+                settings.JWT_SECRET_KEY,
+                algorithms=["HS256"],
+                options={"verify_exp": False},
+            )
+            return decoded_token
+        except Exception as e:
+            return {
+                "message": f"Authentication error: {str(e)}",
+                "error": "invalid_token",
+            }
+    else:
+        return {"sub": "local"}

application/cache.py

@@ -0,0 +1,117 @@
+import json
+import logging
+import time
+from threading import Lock
+
+import redis
+
+from application.core.settings import settings
+from application.utils import get_hash
+
+logger = logging.getLogger(__name__)
+
+_redis_instance = None
+_redis_creation_failed = False
+_instance_lock = Lock()
+
+def get_redis_instance():
+    global _redis_instance, _redis_creation_failed
+    if _redis_instance is None and not _redis_creation_failed:
+        with _instance_lock:
+            if _redis_instance is None and not _redis_creation_failed:
+                try:
+                    _redis_instance = redis.Redis.from_url(
+                        settings.CACHE_REDIS_URL, socket_connect_timeout=2
+                    )
+                except ValueError as e:
+                    logger.error(f"Invalid Redis URL: {e}")
+                    _redis_creation_failed = True  # Stop future attempts
+                    _redis_instance = None
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+                    _redis_instance = None  # Keep trying for connection errors
+    return _redis_instance
+
+
+def gen_cache_key(messages, model="docgpt", tools=None):
+    if not all(isinstance(msg, dict) for msg in messages):
+        raise ValueError("All messages must be dictionaries.")
+    messages_str = json.dumps(messages)
+    tools_str = json.dumps(str(tools)) if tools else ""
+    combined = f"{model}_{messages_str}_{tools_str}"
+    cache_key = get_hash(combined)
+    return cache_key
+
+
+def gen_cache(func):
+    def wrapper(self, model, messages, stream, tools=None, *args, **kwargs):
+        if tools is not None:
+            return func(self, model, messages, stream, tools, *args, **kwargs)
+        
+        try:
+            cache_key = gen_cache_key(messages, model, tools)
+        except ValueError as e:
+            logger.error(f"Cache key generation failed: {e}")
+            return func(self, model, messages, stream, tools, *args, **kwargs)
+
+        redis_client = get_redis_instance()
+        if redis_client:
+            try:
+                cached_response = redis_client.get(cache_key)
+                if cached_response:
+                    return cached_response.decode("utf-8")
+            except Exception as e:
+                logger.error(f"Error getting cached response: {e}", exc_info=True)
+
+        result = func(self, model, messages, stream, tools, *args, **kwargs)
+        if redis_client and isinstance(result, str):
+            try:
+                redis_client.set(cache_key, result, ex=1800)
+            except Exception as e:
+                logger.error(f"Error setting cache: {e}", exc_info=True)
+
+        return result
+
+    return wrapper
+
+
+def stream_cache(func):
+    def wrapper(self, model, messages, stream, tools=None, *args, **kwargs):
+        if tools is not None:
+            yield from func(self, model, messages, stream, tools, *args, **kwargs)
+            return
+        
+        try:
+            cache_key = gen_cache_key(messages, model, tools)
+        except ValueError as e:
+            logger.error(f"Cache key generation failed: {e}")
+            yield from func(self, model, messages, stream, tools, *args, **kwargs)
+            return
+
+        redis_client = get_redis_instance()
+        if redis_client:
+            try:
+                cached_response = redis_client.get(cache_key)
+                if cached_response:
+                    logger.info(f"Cache hit for stream key: {cache_key}")
+                    cached_response = json.loads(cached_response.decode("utf-8"))
+                    for chunk in cached_response:
+                        yield chunk
+                        time.sleep(0.03)  # Simulate streaming delay
+                    return
+            except Exception as e:
+                logger.error(f"Error getting cached stream: {e}", exc_info=True)
+
+        stream_cache_data = []
+        for chunk in func(self, model, messages, stream, tools, *args, **kwargs):
+            yield chunk
+            stream_cache_data.append(str(chunk))
+
+        if redis_client:
+            try:
+                redis_client.set(cache_key, json.dumps(stream_cache_data), ex=1800)
+                logger.info(f"Stream cache saved for key: {cache_key}")
+            except Exception as e:
+                logger.error(f"Error setting stream cache: {e}", exc_info=True)
+
+    return wrapper

application/celery.py

@@ -1,9 +0,0 @@
-from celery import Celery
-from application.core.settings import settings
-
-def make_celery(app_name=__name__):
-    celery = Celery(app_name, broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND)
-    celery.conf.update(settings)
-    return celery
-
-celery = make_celery()

application/celery_init.py

@@ -0,0 +1,23 @@
+from celery import Celery
+from application.core.settings import settings
+from celery.signals import setup_logging
+
+
+def make_celery(app_name=__name__):
+    celery = Celery(
+        app_name,
+        broker=settings.CELERY_BROKER_URL,
+        backend=settings.CELERY_RESULT_BACKEND,
+    )
+    celery.conf.update(settings)
+    return celery
+
+
+@setup_logging.connect
+def config_loggers(*args, **kwargs):
+    from application.core.logging_config import setup_logging
+
+    setup_logging()
+
+
+celery = make_celery()

application/core/logging_config.py

@@ -0,0 +1,22 @@
+from logging.config import dictConfig
+
+def setup_logging():
+    dictConfig({
+        'version': 1,
+        'formatters': {
+            'default': {
+                'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+            }
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "stream": "ext://sys.stdout",
+                "formatter": "default",
+            }
+        },
+        'root': {
+            'level': 'INFO',
+            'handlers': ['console'],
+        },
+    })

application/core/mongo_db.py

@@ -0,0 +1,24 @@
+from application.core.settings import settings
+from pymongo import MongoClient
+
+
+class MongoDB:
+    _client = None
+
+    @classmethod
+    def get_client(cls):
+        """
+        Get the MongoDB client instance, creating it if necessary.
+        """
+        if cls._client is None:
+            cls._client = MongoClient(settings.MONGO_URI)
+        return cls._client
+
+    @classmethod
+    def close_client(cls):
+        """
+        Close the MongoDB client connection.
+        """
+        if cls._client is not None:
+            cls._client.close()
+            cls._client = None

application/core/settings.py

@@ -1,36 +1,120 @@
-from pathlib import Path
 import os
+from pathlib import Path
+from typing import Optional
 
-from pydantic import BaseSettings
-current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from pydantic_settings import BaseSettings
+
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
 
 
 class Settings(BaseSettings):
-    LLM_NAME: str = "openai"
-    EMBEDDINGS_NAME: str = "openai_text-embedding-ada-002"
+    AUTH_TYPE: Optional[str] = None  # simple_jwt, session_jwt, or None
+    LLM_PROVIDER: str = "docsgpt"
+    LLM_NAME: Optional[str] = (
+        None  # if LLM_PROVIDER is openai, LLM_NAME can be gpt-4 or gpt-3.5-turbo
+    )
+    EMBEDDINGS_NAME: str = "huggingface_sentence-transformers/all-mpnet-base-v2"
     CELERY_BROKER_URL: str = "redis://localhost:6379/0"
     CELERY_RESULT_BACKEND: str = "redis://localhost:6379/1"
     MONGO_URI: str = "mongodb://localhost:27017/docsgpt"
-    MODEL_PATH: str = os.path.join(current_dir, "models/docsgpt-7b-f16.gguf")
-    TOKENS_MAX_HISTORY: int = 150
+    MONGO_DB_NAME: str = "docsgpt"
+    LLM_PATH: str = os.path.join(current_dir, "models/docsgpt-7b-f16.gguf")
+    DEFAULT_MAX_HISTORY: int = 150
+    LLM_TOKEN_LIMITS: dict = {
+        "gpt-4o-mini": 128000,
+        "gpt-3.5-turbo": 4096,
+        "claude-2": 1e5,
+        "gemini-2.0-flash-exp": 1e6,
+    }
     UPLOAD_FOLDER: str = "inputs"
-    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch"
+    PARSE_PDF_AS_IMAGE: bool = False
+    PARSE_IMAGE_REMOTE: bool = False
+    VECTOR_STORE: str = (
+        "faiss"  #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb"
+    )
+    RETRIEVERS_ENABLED: list = ["classic_rag"]
+    AGENT_NAME: str = "classic"
+    FALLBACK_LLM_PROVIDER: Optional[str] = None  # provider for fallback llm
+    FALLBACK_LLM_NAME: Optional[str] = None  # model name for fallback llm
+    FALLBACK_LLM_API_KEY: Optional[str] = None  # api key for fallback llm
+
+    # Google Drive integration
+    GOOGLE_CLIENT_ID: Optional[str] = None # Replace with your actual Google OAuth client ID
+    GOOGLE_CLIENT_SECRET: Optional[str] = None# Replace with your actual Google OAuth client secret
+    CONNECTOR_REDIRECT_BASE_URI: Optional[str] = "http://127.0.0.1:7091/api/connectors/callback" 
+    ##append ?provider={provider_name} in your Provider console like http://127.0.0.1:7091/api/connectors/callback?provider=google_drive
+
+
+    # LLM Cache
+    CACHE_REDIS_URL: str = "redis://localhost:6379/2"
 
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
 
-    API_KEY: str = None  # LLM api key
-    EMBEDDINGS_KEY: str = None  # api key for embeddings (if using openai, just copy API_KEY
-    OPENAI_API_BASE: str = None  # azure openai api base url
-    OPENAI_API_VERSION: str = None  # azure openai api version
-    AZURE_DEPLOYMENT_NAME: str = None  # azure deployment name for answering
-    AZURE_EMBEDDINGS_DEPLOYMENT_NAME: str = None  # azure deployment name for embeddings
+    API_KEY: Optional[str] = None  # LLM api key
+    EMBEDDINGS_KEY: Optional[str] = (
+        None  # api key for embeddings (if using openai, just copy API_KEY)
+    )
+    OPENAI_API_BASE: Optional[str] = None  # azure openai api base url
+    OPENAI_API_VERSION: Optional[str] = None  # azure openai api version
+    AZURE_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for answering
+    AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = (
+        None  # azure deployment name for embeddings
+    )
+    OPENAI_BASE_URL: Optional[str] = (
+        None  # openai base url for open ai compatable models
+    )
 
     # elasticsearch
-    ELASTIC_CLOUD_ID: str = None # cloud id for elasticsearch
-    ELASTIC_USERNAME: str = None # username for elasticsearch
-    ELASTIC_PASSWORD: str = None # password for elasticsearch
-    ELASTIC_URL: str = None # url for elasticsearch
-    ELASTIC_INDEX: str = "docsgpt" # index name for elasticsearch
+    ELASTIC_CLOUD_ID: Optional[str] = None  # cloud id for elasticsearch
+    ELASTIC_USERNAME: Optional[str] = None  # username for elasticsearch
+    ELASTIC_PASSWORD: Optional[str] = None  # password for elasticsearch
+    ELASTIC_URL: Optional[str] = None  # url for elasticsearch
+    ELASTIC_INDEX: Optional[str] = "docsgpt"  # index name for elasticsearch
+
+    # SageMaker config
+    SAGEMAKER_ENDPOINT: Optional[str] = None  # SageMaker endpoint name
+    SAGEMAKER_REGION: Optional[str] = None  # SageMaker region name
+    SAGEMAKER_ACCESS_KEY: Optional[str] = None  # SageMaker access key
+    SAGEMAKER_SECRET_KEY: Optional[str] = None  # SageMaker secret key
+
+    # prem ai project id
+    PREMAI_PROJECT_ID: Optional[str] = None
+
+    # Qdrant vectorstore config
+    QDRANT_COLLECTION_NAME: Optional[str] = "docsgpt"
+    QDRANT_LOCATION: Optional[str] = None
+    QDRANT_URL: Optional[str] = None
+    QDRANT_PORT: Optional[int] = 6333
+    QDRANT_GRPC_PORT: int = 6334
+    QDRANT_PREFER_GRPC: bool = False
+    QDRANT_HTTPS: Optional[bool] = None
+    QDRANT_API_KEY: Optional[str] = None
+    QDRANT_PREFIX: Optional[str] = None
+    QDRANT_TIMEOUT: Optional[float] = None
+    QDRANT_HOST: Optional[str] = None
+    QDRANT_PATH: Optional[str] = None
+    QDRANT_DISTANCE_FUNC: str = "Cosine"
+    
+    # PGVector vectorstore config
+    PGVECTOR_CONNECTION_STRING: Optional[str] = None
+    # Milvus vectorstore config
+    MILVUS_COLLECTION_NAME: Optional[str] = "docsgpt"
+    MILVUS_URI: Optional[str] = "./milvus_local.db"  # milvus lite version as default
+    MILVUS_TOKEN: Optional[str] = ""
+
+    # LanceDB vectorstore config
+    LANCEDB_PATH: str = "/tmp/lancedb"  # Path where LanceDB stores its local data
+    LANCEDB_TABLE_NAME: Optional[str] = (
+        "docsgpts"  # Name of the table to use for storing vectors
+    )
+
+    FLASK_DEBUG_MODE: bool = False
+    STORAGE_TYPE: str = "local"  # local or s3
+    URL_STRATEGY: str = "backend"  # backend or s3
+
+    JWT_SECRET_KEY: str = ""
 
 
 path = Path(__file__).parent.parent.absolute()

application/llm/anthropic.py

@@ -0,0 +1,50 @@
+from application.llm.base import BaseLLM
+from application.core.settings import settings
+
+
+class AnthropicLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from anthropic import Anthropic, HUMAN_PROMPT, AI_PROMPT
+
+        super().__init__(*args, **kwargs)
+        self.api_key = (
+            api_key or settings.ANTHROPIC_API_KEY
+        )  # If not provided, use a default from settings
+        self.user_api_key = user_api_key
+        self.anthropic = Anthropic(api_key=self.api_key)
+        self.HUMAN_PROMPT = HUMAN_PROMPT
+        self.AI_PROMPT = AI_PROMPT
+
+    def _raw_gen(
+        self, baseself, model, messages, stream=False, tools=None, max_tokens=300, **kwargs
+    ):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
+        prompt = f"### Context \n {context} \n ### Question \n {user_question}"
+        if stream:
+            return self.gen_stream(model, prompt, stream, max_tokens, **kwargs)
+
+        completion = self.anthropic.completions.create(
+            model=model,
+            max_tokens_to_sample=max_tokens,
+            stream=stream,
+            prompt=f"{self.HUMAN_PROMPT} {prompt}{self.AI_PROMPT}",
+        )
+        return completion.completion
+
+    def _raw_gen_stream(
+        self, baseself, model, messages, stream=True, tools=None, max_tokens=300, **kwargs
+    ):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
+        prompt = f"### Context \n {context} \n ### Question \n {user_question}"
+        stream_response = self.anthropic.completions.create(
+            model=model,
+            prompt=f"{self.HUMAN_PROMPT} {prompt}{self.AI_PROMPT}",
+            max_tokens_to_sample=max_tokens,
+            stream=True,
+        )
+
+        for completion in stream_response:
+            yield completion.completion

application/llm/base.py

@@ -1,14 +1,144 @@
+import logging
 from abc import ABC, abstractmethod
 
+from application.cache import gen_cache, stream_cache
+
+from application.core.settings import settings
+from application.usage import gen_token_usage, stream_token_usage
+
+logger = logging.getLogger(__name__)
+
 
 class BaseLLM(ABC):
-    def __init__(self):
+    def __init__(
+        self,
+        decoded_token=None,
+    ):
+        self.decoded_token = decoded_token
+        self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
+        self.fallback_provider = settings.FALLBACK_LLM_PROVIDER
+        self.fallback_model_name = settings.FALLBACK_LLM_NAME
+        self.fallback_llm_api_key = settings.FALLBACK_LLM_API_KEY
+        self._fallback_llm = None
+
+    @property
+    def fallback_llm(self):
+        """Lazy-loaded fallback LLM instance."""
+        if (
+            self._fallback_llm is None
+            and self.fallback_provider
+            and self.fallback_model_name
+        ):
+            try:
+                from application.llm.llm_creator import LLMCreator
+
+                self._fallback_llm = LLMCreator.create_llm(
+                    self.fallback_provider,
+                    self.fallback_llm_api_key,
+                    None,
+                    self.decoded_token,
+                )
+            except Exception as e:
+                logger.error(
+                    f"Failed to initialize fallback LLM: {str(e)}", exc_info=True
+                )
+        return self._fallback_llm
+
+    def _execute_with_fallback(
+        self, method_name: str, decorators: list, *args, **kwargs
+    ):
+        """
+        Unified method execution with fallback support.
+
+        Args:
+            method_name: Name of the raw method ('_raw_gen' or '_raw_gen_stream')
+            decorators: List of decorators to apply
+            *args: Positional arguments
+            **kwargs: Keyword arguments
+        """
+
+        def decorated_method():
+            method = getattr(self, method_name)
+            for decorator in decorators:
+                method = decorator(method)
+            return method(self, *args, **kwargs)
+
+        try:
+            return decorated_method()
+        except Exception as e:
+            if not self.fallback_llm:
+                logger.error(f"Primary LLM failed and no fallback available: {str(e)}")
+                raise
+            logger.warning(
+                f"Falling back to {self.fallback_provider}/{self.fallback_model_name}. Error: {str(e)}"
+            )
+
+            fallback_method = getattr(
+                self.fallback_llm, method_name.replace("_raw_", "")
+            )
+            return fallback_method(*args, **kwargs)
+
+    def gen(self, model, messages, stream=False, tools=None, *args, **kwargs):
+        decorators = [gen_token_usage, gen_cache]
+        return self._execute_with_fallback(
+            "_raw_gen",
+            decorators,
+            model=model,
+            messages=messages,
+            stream=stream,
+            tools=tools,
+            *args,
+            **kwargs,
+        )
+
+    def gen_stream(self, model, messages, stream=True, tools=None, *args, **kwargs):
+        decorators = [stream_cache, stream_token_usage]
+        return self._execute_with_fallback(
+            "_raw_gen_stream",
+            decorators,
+            model=model,
+            messages=messages,
+            stream=stream,
+            tools=tools,
+            *args,
+            **kwargs,
+        )
+
+    @abstractmethod
+    def _raw_gen(self, model, messages, stream, tools, *args, **kwargs):
         pass
 
     @abstractmethod
-    def gen(self, *args, **kwargs):
+    def _raw_gen_stream(self, model, messages, stream, *args, **kwargs):
         pass
 
-    @abstractmethod
-    def gen_stream(self, *args, **kwargs):
-        pass
+    def supports_tools(self):
+        return hasattr(self, "_supports_tools") and callable(
+            getattr(self, "_supports_tools")
+        )
+
+    def _supports_tools(self):
+        raise NotImplementedError("Subclass must implement _supports_tools method")
+
+    def supports_structured_output(self):
+        """Check if the LLM supports structured output/JSON schema enforcement"""
+        return hasattr(self, "_supports_structured_output") and callable(
+            getattr(self, "_supports_structured_output")
+        )
+
+    def _supports_structured_output(self):
+        return False
+
+    def prepare_structured_output_format(self, json_schema):
+        """Prepare structured output format specific to the LLM provider"""
+        _ = json_schema
+        return None
+
+    def get_supported_attachment_types(self):
+        """
+        Return a list of MIME types supported by this LLM for file uploads.
+
+        Returns:
+            list: List of supported MIME types
+        """
+        return []

application/llm/docsgpt_provider.py

@@ -0,0 +1,131 @@
+import json
+
+from application.core.settings import settings
+from application.llm.base import BaseLLM
+
+
+class DocsGPTAPILLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from openai import OpenAI
+
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key="sk-docsgpt-public", base_url="https://oai.arc53.com")
+        self.user_api_key = user_api_key
+        self.api_key = api_key
+
+    def _clean_messages_openai(self, messages):
+        cleaned_messages = []
+        for message in messages:
+            role = message.get("role")
+            content = message.get("content")
+
+            if role == "model":
+                role = "assistant"
+
+            if role and content is not None:
+                if isinstance(content, str):
+                    cleaned_messages.append({"role": role, "content": content})
+                elif isinstance(content, list):
+                    for item in content:
+                        if "text" in item:
+                            cleaned_messages.append(
+                                {"role": role, "content": item["text"]}
+                            )
+                        elif "function_call" in item:
+                            tool_call = {
+                                "id": item["function_call"]["call_id"],
+                                "type": "function",
+                                "function": {
+                                    "name": item["function_call"]["name"],
+                                    "arguments": json.dumps(
+                                        item["function_call"]["args"]
+                                    ),
+                                },
+                            }
+                            cleaned_messages.append(
+                                {
+                                    "role": "assistant",
+                                    "content": None,
+                                    "tool_calls": [tool_call],
+                                }
+                            )
+                        elif "function_response" in item:
+                            cleaned_messages.append(
+                                {
+                                    "role": "tool",
+                                    "tool_call_id": item["function_response"][
+                                        "call_id"
+                                    ],
+                                    "content": json.dumps(
+                                        item["function_response"]["response"]["result"]
+                                    ),
+                                }
+                            )
+                        else:
+                            raise ValueError(
+                                f"Unexpected content dictionary format: {item}"
+                            )
+                else:
+                    raise ValueError(f"Unexpected content type: {type(content)}")
+
+        return cleaned_messages
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        tools=None,
+        engine=settings.AZURE_DEPLOYMENT_NAME,
+        **kwargs,
+    ):
+        messages = self._clean_messages_openai(messages)
+        if tools:
+            response = self.client.chat.completions.create(
+                model="docsgpt",
+                messages=messages,
+                stream=stream,
+                tools=tools,
+                **kwargs,
+            )
+            return response.choices[0]
+        else:
+            response = self.client.chat.completions.create(
+                model="docsgpt", messages=messages, stream=stream, **kwargs
+            )
+            return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        tools=None,
+        engine=settings.AZURE_DEPLOYMENT_NAME,
+        **kwargs,
+    ):
+        messages = self._clean_messages_openai(messages)
+        if tools:
+            response = self.client.chat.completions.create(
+                model="docsgpt",
+                messages=messages,
+                stream=stream,
+                tools=tools,
+                **kwargs,
+            )
+        else:
+            response = self.client.chat.completions.create(
+                model="docsgpt", messages=messages, stream=stream, **kwargs
+            )
+
+        for line in response:
+            if len(line.choices) > 0 and line.choices[0].delta.content is not None and len(line.choices[0].delta.content) > 0:
+                yield line.choices[0].delta.content
+            elif len(line.choices) > 0:
+                yield line.choices[0]
+
+    def _supports_tools(self):
+        return True

application/llm/google_ai.py

@@ -0,0 +1,406 @@
+import json
+import logging
+
+from google import genai
+from google.genai import types
+
+from application.core.settings import settings
+
+from application.llm.base import BaseLLM
+from application.storage.storage_creator import StorageCreator
+
+
+class GoogleLLM(BaseLLM):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.client = genai.Client(api_key=self.api_key)
+        self.storage = StorageCreator.get_storage()
+
+    def get_supported_attachment_types(self):
+        """
+        Return a list of MIME types supported by Google Gemini for file uploads.
+
+        Returns:
+            list: List of supported MIME types
+        """
+        return [
+            "application/pdf",
+            "image/png",
+            "image/jpeg",
+            "image/jpg",
+            "image/webp",
+            "image/gif",
+        ]
+
+    def prepare_messages_with_attachments(self, messages, attachments=None):
+        """
+        Process attachments using Google AI's file API for more efficient handling.
+
+        Args:
+            messages (list): List of message dictionaries.
+            attachments (list): List of attachment dictionaries with content and metadata.
+
+        Returns:
+            list: Messages formatted with file references for Google AI API.
+        """
+        if not attachments:
+            return messages
+
+        prepared_messages = messages.copy()
+
+        # Find the user message to attach files to the last one
+        user_message_index = None
+        for i in range(len(prepared_messages) - 1, -1, -1):
+            if prepared_messages[i].get("role") == "user":
+                user_message_index = i
+                break
+
+        if user_message_index is None:
+            user_message = {"role": "user", "content": []}
+            prepared_messages.append(user_message)
+            user_message_index = len(prepared_messages) - 1
+
+        if isinstance(prepared_messages[user_message_index].get("content"), str):
+            text_content = prepared_messages[user_message_index]["content"]
+            prepared_messages[user_message_index]["content"] = [
+                {"type": "text", "text": text_content}
+            ]
+        elif not isinstance(prepared_messages[user_message_index].get("content"), list):
+            prepared_messages[user_message_index]["content"] = []
+
+        files = []
+        for attachment in attachments:
+            mime_type = attachment.get("mime_type")
+
+            if mime_type in self.get_supported_attachment_types():
+                try:
+                    file_uri = self._upload_file_to_google(attachment)
+                    logging.info(
+                        f"GoogleLLM: Successfully uploaded file, got URI: {file_uri}"
+                    )
+                    files.append({"file_uri": file_uri, "mime_type": mime_type})
+                except Exception as e:
+                    logging.error(
+                        f"GoogleLLM: Error uploading file: {e}", exc_info=True
+                    )
+                    if "content" in attachment:
+                        prepared_messages[user_message_index]["content"].append(
+                            {
+                                "type": "text",
+                                "text": f"[File could not be processed: {attachment.get('path', 'unknown')}]",
+                            }
+                        )
+
+        if files:
+            logging.info(f"GoogleLLM: Adding {len(files)} files to message")
+            prepared_messages[user_message_index]["content"].append({"files": files})
+
+        return prepared_messages
+
+    def _upload_file_to_google(self, attachment):
+        """
+        Upload a file to Google AI and return the file URI.
+
+        Args:
+            attachment (dict): Attachment dictionary with path and metadata.
+
+        Returns:
+            str: Google AI file URI for the uploaded file.
+        """
+        if "google_file_uri" in attachment:
+            return attachment["google_file_uri"]
+
+        file_path = attachment.get("path")
+        if not file_path:
+            raise ValueError("No file path provided in attachment")
+
+        if not self.storage.file_exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        try:
+            file_uri = self.storage.process_file(
+                file_path,
+                lambda local_path, **kwargs: self.client.files.upload(
+                    file=local_path
+                ).uri,
+            )
+
+            from application.core.mongo_db import MongoDB
+
+            mongo = MongoDB.get_client()
+            db = mongo[settings.MONGO_DB_NAME]
+            attachments_collection = db["attachments"]
+            if "_id" in attachment:
+                attachments_collection.update_one(
+                    {"_id": attachment["_id"]}, {"$set": {"google_file_uri": file_uri}}
+                )
+
+            return file_uri
+        except Exception as e:
+            logging.error(f"Error uploading file to Google AI: {e}", exc_info=True)
+            raise
+
+    def _clean_messages_google(self, messages):
+        cleaned_messages = []
+        for message in messages:
+            role = message.get("role")
+            content = message.get("content")
+
+            if role == "assistant":
+                role = "model"
+
+            parts = []
+            if role and content is not None:
+                if isinstance(content, str):
+                    parts = [types.Part.from_text(text=content)]
+                elif isinstance(content, list):
+                    for item in content:
+                        if "text" in item:
+                            parts.append(types.Part.from_text(text=item["text"]))
+                        elif "function_call" in item:
+                            parts.append(
+                                types.Part.from_function_call(
+                                    name=item["function_call"]["name"],
+                                    args=item["function_call"]["args"],
+                                )
+                            )
+                        elif "function_response" in item:
+                            parts.append(
+                                types.Part.from_function_response(
+                                    name=item["function_response"]["name"],
+                                    response=item["function_response"]["response"],
+                                )
+                            )
+                        elif "files" in item:
+                            for file_data in item["files"]:
+                                parts.append(
+                                    types.Part.from_uri(
+                                        file_uri=file_data["file_uri"],
+                                        mime_type=file_data["mime_type"],
+                                    )
+                                )
+                        else:
+                            raise ValueError(
+                                f"Unexpected content dictionary format:{item}"
+                            )
+                else:
+                    raise ValueError(f"Unexpected content type: {type(content)}")
+
+                cleaned_messages.append(types.Content(role=role, parts=parts))
+
+        return cleaned_messages
+
+    def _clean_tools_format(self, tools_list):
+        genai_tools = []
+        for tool_data in tools_list:
+            if tool_data["type"] == "function":
+                function = tool_data["function"]
+                parameters = function["parameters"]
+                properties = parameters.get("properties", {})
+
+                if properties:
+                    genai_function = dict(
+                        name=function["name"],
+                        description=function["description"],
+                        parameters={
+                            "type": "OBJECT",
+                            "properties": {
+                                k: {
+                                    **v,
+                                    "type": v["type"].upper() if v["type"] else None,
+                                }
+                                for k, v in properties.items()
+                            },
+                            "required": (
+                                parameters["required"]
+                                if "required" in parameters
+                                else []
+                            ),
+                        },
+                    )
+                else:
+                    genai_function = dict(
+                        name=function["name"],
+                        description=function["description"],
+                    )
+
+                genai_tool = types.Tool(function_declarations=[genai_function])
+                genai_tools.append(genai_tool)
+
+        return genai_tools
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        tools=None,
+        formatting="openai",
+        response_schema=None,
+        **kwargs,
+    ):
+        client = genai.Client(api_key=self.api_key)
+        if formatting == "openai":
+            messages = self._clean_messages_google(messages)
+        config = types.GenerateContentConfig()
+        if messages[0].role == "system":
+            config.system_instruction = messages[0].parts[0].text
+            messages = messages[1:]
+
+        if tools:
+            cleaned_tools = self._clean_tools_format(tools)
+            config.tools = cleaned_tools
+
+        # Add response schema for structured output if provided
+        if response_schema:
+            config.response_schema = response_schema
+            config.response_mime_type = "application/json"
+
+        response = client.models.generate_content(
+            model=model,
+            contents=messages,
+            config=config,
+        )
+
+        if tools:
+            return response
+        else:
+            return response.text
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        tools=None,
+        formatting="openai",
+        response_schema=None,
+        **kwargs,
+    ):
+        client = genai.Client(api_key=self.api_key)
+        if formatting == "openai":
+            messages = self._clean_messages_google(messages)
+        config = types.GenerateContentConfig()
+        if messages[0].role == "system":
+            config.system_instruction = messages[0].parts[0].text
+            messages = messages[1:]
+
+        if tools:
+            cleaned_tools = self._clean_tools_format(tools)
+            config.tools = cleaned_tools
+
+        # Add response schema for structured output if provided
+        if response_schema:
+            config.response_schema = response_schema
+            config.response_mime_type = "application/json"
+
+        # Check if we have both tools and file attachments
+        has_attachments = False
+        for message in messages:
+            for part in message.parts:
+                if hasattr(part, "file_data") and part.file_data is not None:
+                    has_attachments = True
+                    break
+            if has_attachments:
+                break
+
+        logging.info(
+            f"GoogleLLM: Starting stream generation. Model: {model}, Messages: {json.dumps(messages, default=str)}, Has attachments: {has_attachments}"
+        )
+
+        response = client.models.generate_content_stream(
+            model=model,
+            contents=messages,
+            config=config,
+        )
+
+        for chunk in response:
+            if hasattr(chunk, "candidates") and chunk.candidates:
+                for candidate in chunk.candidates:
+                    if candidate.content and candidate.content.parts:
+                        for part in candidate.content.parts:
+                            if part.function_call:
+                                yield part
+                            elif part.text:
+                                yield part.text
+            elif hasattr(chunk, "text"):
+                yield chunk.text
+
+    def _supports_tools(self):
+        return True
+
+    def _supports_structured_output(self):
+        return True
+
+    def prepare_structured_output_format(self, json_schema):
+        if not json_schema:
+            return None
+
+        type_map = {
+            "object": "OBJECT",
+            "array": "ARRAY",
+            "string": "STRING",
+            "integer": "INTEGER",
+            "number": "NUMBER",
+            "boolean": "BOOLEAN",
+        }
+
+        def convert(schema):
+            if not isinstance(schema, dict):
+                return schema
+
+            result = {}
+            schema_type = schema.get("type")
+            if schema_type:
+                result["type"] = type_map.get(schema_type.lower(), schema_type.upper())
+
+            for key in [
+                "description",
+                "nullable",
+                "enum",
+                "minItems",
+                "maxItems",
+                "required",
+                "propertyOrdering",
+            ]:
+                if key in schema:
+                    result[key] = schema[key]
+
+            if "format" in schema:
+                format_value = schema["format"]
+                if schema_type == "string":
+                    if format_value == "date":
+                        result["format"] = "date-time"
+                    elif format_value in ["enum", "date-time"]:
+                        result["format"] = format_value
+                else:
+                    result["format"] = format_value
+
+            if "properties" in schema:
+                result["properties"] = {
+                    k: convert(v) for k, v in schema["properties"].items()
+                }
+                if "propertyOrdering" not in result and result.get("type") == "OBJECT":
+                    result["propertyOrdering"] = list(result["properties"].keys())
+
+            if "items" in schema:
+                result["items"] = convert(schema["items"])
+
+            for field in ["anyOf", "oneOf", "allOf"]:
+                if field in schema:
+                    result[field] = [convert(s) for s in schema[field]]
+
+            return result
+
+        try:
+            return convert(json_schema)
+        except Exception as e:
+            logging.error(
+                f"Error preparing structured output format for Google: {e}",
+                exc_info=True,
+            )
+            return None

application/llm/groq.py

@@ -0,0 +1,32 @@
+from application.llm.base import BaseLLM
+from openai import OpenAI
+
+
+class GroqLLM(BaseLLM):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs):
+        if tools:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, tools=tools, **kwargs
+            )
+            return response.choices[0]
+        else:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, **kwargs
+            )
+            return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self, baseself, model, messages, stream=True, tools=None, **kwargs
+    ):
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+        for line in response:
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content

application/llm/handlers/base.py

@@ -0,0 +1,335 @@
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from application.logging import build_stack_data
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ToolCall:
+    """Represents a tool/function call from the LLM."""
+
+    id: str
+    name: str
+    arguments: Union[str, Dict]
+    index: Optional[int] = None
+
+    @classmethod
+    def from_dict(cls, data: Dict) -> "ToolCall":
+        """Create ToolCall from dictionary."""
+        return cls(
+            id=data.get("id", ""),
+            name=data.get("name", ""),
+            arguments=data.get("arguments", {}),
+            index=data.get("index"),
+        )
+
+
+@dataclass
+class LLMResponse:
+    """Represents a response from the LLM."""
+
+    content: str
+    tool_calls: List[ToolCall]
+    finish_reason: str
+    raw_response: Any
+
+    @property
+    def requires_tool_call(self) -> bool:
+        """Check if the response requires tool calls."""
+        return bool(self.tool_calls) and self.finish_reason == "tool_calls"
+
+
+class LLMHandler(ABC):
+    """Abstract base class for LLM handlers."""
+
+    def __init__(self):
+        self.llm_calls = []
+        self.tool_calls = []
+
+    @abstractmethod
+    def parse_response(self, response: Any) -> LLMResponse:
+        """Parse raw LLM response into standardized format."""
+        pass
+
+    @abstractmethod
+    def create_tool_message(self, tool_call: ToolCall, result: Any) -> Dict:
+        """Create a tool result message for the conversation history."""
+        pass
+
+    @abstractmethod
+    def _iterate_stream(self, response: Any) -> Generator:
+        """Iterate through streaming response chunks."""
+        pass
+
+    def process_message_flow(
+        self,
+        agent,
+        initial_response,
+        tools_dict: Dict,
+        messages: List[Dict],
+        attachments: Optional[List] = None,
+        stream: bool = False,
+    ) -> Union[str, Generator]:
+        """
+        Main orchestration method for processing LLM message flow.
+
+        Args:
+            agent: The agent instance
+            initial_response: Initial LLM response
+            tools_dict: Dictionary of available tools
+            messages: Conversation history
+            attachments: Optional attachments
+            stream: Whether to use streaming
+
+        Returns:
+            Final response or generator for streaming
+        """
+        messages = self.prepare_messages(agent, messages, attachments)
+
+        if stream:
+            return self.handle_streaming(agent, initial_response, tools_dict, messages)
+        else:
+            return self.handle_non_streaming(
+                agent, initial_response, tools_dict, messages
+            )
+
+    def prepare_messages(
+        self, agent, messages: List[Dict], attachments: Optional[List] = None
+    ) -> List[Dict]:
+        """
+        Prepare messages with attachments and provider-specific formatting.
+
+        Args:
+            agent: The agent instance
+            messages: Original messages
+            attachments: List of attachments
+
+        Returns:
+            Prepared messages list
+        """
+        if not attachments:
+            return messages
+        logger.info(f"Preparing messages with {len(attachments)} attachments")
+        supported_types = agent.llm.get_supported_attachment_types()
+
+        supported_attachments = [
+            a for a in attachments if a.get("mime_type") in supported_types
+        ]
+        unsupported_attachments = [
+            a for a in attachments if a.get("mime_type") not in supported_types
+        ]
+
+        # Process supported attachments with the LLM's custom method
+
+        if supported_attachments:
+            logger.info(
+                f"Processing {len(supported_attachments)} supported attachments"
+            )
+            messages = agent.llm.prepare_messages_with_attachments(
+                messages, supported_attachments
+            )
+        # Process unsupported attachments with default method
+
+        if unsupported_attachments:
+            logger.info(
+                f"Processing {len(unsupported_attachments)} unsupported attachments"
+            )
+            messages = self._append_unsupported_attachments(
+                messages, unsupported_attachments
+            )
+        return messages
+
+    def _append_unsupported_attachments(
+        self, messages: List[Dict], attachments: List[Dict]
+    ) -> List[Dict]:
+        """
+        Default method to append unsupported attachment content to system prompt.
+
+        Args:
+            messages: Current messages
+            attachments: List of unsupported attachments
+
+        Returns:
+            Updated messages list
+        """
+        prepared_messages = messages.copy()
+        attachment_texts = []
+
+        for attachment in attachments:
+            logger.info(f"Adding attachment {attachment.get('id')} to context")
+            if "content" in attachment:
+                attachment_texts.append(
+                    f"Attached file content:\n\n{attachment['content']}"
+                )
+        if attachment_texts:
+            combined_text = "\n\n".join(attachment_texts)
+
+            system_msg = next(
+                (msg for msg in prepared_messages if msg.get("role") == "system"),
+                {"role": "system", "content": ""},
+            )
+
+            if system_msg not in prepared_messages:
+                prepared_messages.insert(0, system_msg)
+            system_msg["content"] += f"\n\n{combined_text}"
+        return prepared_messages
+
+    def handle_tool_calls(
+        self, agent, tool_calls: List[ToolCall], tools_dict: Dict, messages: List[Dict]
+    ) -> Generator:
+        """
+        Execute tool calls and update conversation history.
+
+        Args:
+            agent: The agent instance
+            tool_calls: List of tool calls to execute
+            tools_dict: Available tools dictionary
+            messages: Current conversation history
+
+        Returns:
+            Updated messages list
+        """
+        updated_messages = messages.copy()
+
+        for call in tool_calls:
+            try:
+                self.tool_calls.append(call)
+                tool_executor_gen = agent._execute_tool_action(tools_dict, call)
+                while True:
+                    try:
+                        yield next(tool_executor_gen)
+                    except StopIteration as e:
+                        tool_response, call_id = e.value
+                        break
+
+                updated_messages.append(
+                    {
+                        "role": "assistant",
+                        "content": [
+                            {
+                                "function_call": {
+                                    "name": call.name,
+                                    "args": call.arguments,
+                                    "call_id": call_id,
+                                }
+                            }
+                        ],
+                    }
+                )
+
+                updated_messages.append(self.create_tool_message(call, tool_response))
+
+            except Exception as e:
+                logger.error(f"Error executing tool: {str(e)}", exc_info=True)
+                updated_messages.append(
+                    {
+                        "role": "tool",
+                        "content": f"Error executing tool: {str(e)}",
+                        "tool_call_id": call.id,
+                    }
+                )
+
+        return updated_messages
+
+    def handle_non_streaming(
+        self, agent, response: Any, tools_dict: Dict, messages: List[Dict]
+    ) -> Generator:
+        """
+        Handle non-streaming response flow.
+
+        Args:
+            agent: The agent instance
+            response: Current LLM response
+            tools_dict: Available tools dictionary
+            messages: Conversation history
+
+        Returns:
+            Final response after processing all tool calls
+        """
+        parsed = self.parse_response(response)
+        self.llm_calls.append(build_stack_data(agent.llm))
+
+        while parsed.requires_tool_call:
+            tool_handler_gen = self.handle_tool_calls(
+                agent, parsed.tool_calls, tools_dict, messages
+            )
+            while True:
+                try:
+                    yield next(tool_handler_gen)
+                except StopIteration as e:
+                    messages = e.value
+                    break
+
+            response = agent.llm.gen(
+                model=agent.gpt_model, messages=messages, tools=agent.tools
+            )
+            parsed = self.parse_response(response)
+            self.llm_calls.append(build_stack_data(agent.llm))
+
+        return parsed.content
+
+    def handle_streaming(
+        self, agent, response: Any, tools_dict: Dict, messages: List[Dict]
+    ) -> Generator:
+        """
+        Handle streaming response flow.
+
+        Args:
+            agent: The agent instance
+            response: Current LLM response
+            tools_dict: Available tools dictionary
+            messages: Conversation history
+
+        Yields:
+            Streaming response chunks
+        """
+        buffer = ""
+        tool_calls = {}
+
+        for chunk in self._iterate_stream(response):
+            if isinstance(chunk, str):
+                yield chunk
+                continue
+            parsed = self.parse_response(chunk)
+
+            if parsed.tool_calls:
+                for call in parsed.tool_calls:
+                    if call.index not in tool_calls:
+                        tool_calls[call.index] = call
+                    else:
+                        existing = tool_calls[call.index]
+                        if call.id:
+                            existing.id = call.id
+                        if call.name:
+                            existing.name = call.name
+                        if call.arguments:
+                            existing.arguments += call.arguments
+            if parsed.finish_reason == "tool_calls":
+                tool_handler_gen = self.handle_tool_calls(
+                    agent, list(tool_calls.values()), tools_dict, messages
+                )
+                while True:
+                    try:
+                        yield next(tool_handler_gen)
+                    except StopIteration as e:
+                        messages = e.value
+                        break
+                tool_calls = {}
+
+                response = agent.llm.gen_stream(
+                    model=agent.gpt_model, messages=messages, tools=agent.tools
+                )
+                self.llm_calls.append(build_stack_data(agent.llm))
+
+                yield from self.handle_streaming(agent, response, tools_dict, messages)
+                return
+            if parsed.content:
+                buffer += parsed.content
+                yield buffer
+                buffer = ""
+            if parsed.finish_reason == "stop":
+                return

application/llm/handlers/google.py

@@ -0,0 +1,78 @@
+import uuid
+from typing import Any, Dict, Generator
+
+from application.llm.handlers.base import LLMHandler, LLMResponse, ToolCall
+
+
+class GoogleLLMHandler(LLMHandler):
+    """Handler for Google's GenAI API."""
+
+    def parse_response(self, response: Any) -> LLMResponse:
+        """Parse Google response into standardized format."""
+
+        if isinstance(response, str):
+            return LLMResponse(
+                content=response,
+                tool_calls=[],
+                finish_reason="stop",
+                raw_response=response,
+            )
+
+        if hasattr(response, "candidates"):
+            parts = response.candidates[0].content.parts if response.candidates else []
+            tool_calls = [
+                ToolCall(
+                    id=str(uuid.uuid4()),
+                    name=part.function_call.name,
+                    arguments=part.function_call.args,
+                )
+                for part in parts
+                if hasattr(part, "function_call") and part.function_call is not None
+            ]
+
+            content = " ".join(
+                part.text
+                for part in parts
+                if hasattr(part, "text") and part.text is not None
+            )
+            return LLMResponse(
+                content=content,
+                tool_calls=tool_calls,
+                finish_reason="tool_calls" if tool_calls else "stop",
+                raw_response=response,
+            )
+
+        else:
+            tool_calls = []
+            if hasattr(response, "function_call"):
+                tool_calls.append(
+                    ToolCall(
+                        id=str(uuid.uuid4()),
+                        name=response.function_call.name,
+                        arguments=response.function_call.args,
+                    )
+                )
+            return LLMResponse(
+                content=response.text if hasattr(response, "text") else "",
+                tool_calls=tool_calls,
+                finish_reason="tool_calls" if tool_calls else "stop",
+                raw_response=response,
+            )
+
+    def create_tool_message(self, tool_call: ToolCall, result: Any) -> Dict:
+        """Create Google-style tool message."""
+        from google.genai import types
+
+        return {
+            "role": "tool",
+            "content": [
+                types.Part.from_function_response(
+                    name=tool_call.name, response={"result": result}
+                ).to_json_dict()
+            ],
+        }
+
+    def _iterate_stream(self, response: Any) -> Generator:
+        """Iterate through Google streaming response."""
+        for chunk in response:
+            yield chunk

application/llm/handlers/handler_creator.py

@@ -0,0 +1,18 @@
+from application.llm.handlers.base import LLMHandler
+from application.llm.handlers.google import GoogleLLMHandler
+from application.llm.handlers.openai import OpenAILLMHandler
+
+
+class LLMHandlerCreator:
+    handlers = {
+        "openai": OpenAILLMHandler,
+        "google": GoogleLLMHandler,
+        "default": OpenAILLMHandler,
+    }
+
+    @classmethod
+    def create_handler(cls, llm_type: str, *args, **kwargs) -> LLMHandler:
+        handler_class = cls.handlers.get(llm_type.lower())
+        if not handler_class:
+            handler_class = OpenAILLMHandler
+        return handler_class(*args, **kwargs)

application/llm/handlers/openai.py

@@ -0,0 +1,57 @@
+from typing import Any, Dict, Generator
+
+from application.llm.handlers.base import LLMHandler, LLMResponse, ToolCall
+
+
+class OpenAILLMHandler(LLMHandler):
+    """Handler for OpenAI API."""
+
+    def parse_response(self, response: Any) -> LLMResponse:
+        """Parse OpenAI response into standardized format."""
+        if isinstance(response, str):
+            return LLMResponse(
+                content=response,
+                tool_calls=[],
+                finish_reason="stop",
+                raw_response=response,
+            )
+
+        message = getattr(response, "message", None) or getattr(response, "delta", None)
+
+        tool_calls = []
+        if hasattr(message, "tool_calls"):
+            tool_calls = [
+                ToolCall(
+                    id=getattr(tc, "id", ""),
+                    name=getattr(tc.function, "name", ""),
+                    arguments=getattr(tc.function, "arguments", ""),
+                    index=getattr(tc, "index", None),
+                )
+                for tc in message.tool_calls or []
+            ]
+        return LLMResponse(
+            content=getattr(message, "content", ""),
+            tool_calls=tool_calls,
+            finish_reason=getattr(response, "finish_reason", ""),
+            raw_response=response,
+        )
+
+    def create_tool_message(self, tool_call: ToolCall, result: Any) -> Dict:
+        """Create OpenAI-style tool message."""
+        return {
+            "role": "tool",
+            "content": [
+                {
+                    "function_response": {
+                        "name": tool_call.name,
+                        "response": {"result": result},
+                        "call_id": tool_call.id,
+                    }
+                }
+            ],
+        }
+
+    def _iterate_stream(self, response: Any) -> Generator:
+        """Iterate through OpenAI streaming response."""
+        for chunk in response:
+            yield chunk

application/llm/huggingface.py

@@ -1,31 +1,68 @@
 from application.llm.base import BaseLLM
 
+
 class HuggingFaceLLM(BaseLLM):
 
-    def __init__(self, api_key, llm_name='Arc53/DocsGPT-7B'):
+    def __init__(
+        self,
+        api_key=None,
+        user_api_key=None,
+        llm_name="Arc53/DocsGPT-7B",
+        q=False,
+        *args,
+        **kwargs,
+    ):
         global hf
 
         from langchain.llms import HuggingFacePipeline
-        from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-        tokenizer = AutoTokenizer.from_pretrained(llm_name)
-        model = AutoModelForCausalLM.from_pretrained(llm_name)
+
+        if q:
+            import torch
+            from transformers import (
+                AutoModelForCausalLM,
+                AutoTokenizer,
+                pipeline,
+                BitsAndBytesConfig,
+            )
+
+            tokenizer = AutoTokenizer.from_pretrained(llm_name)
+            bnb_config = BitsAndBytesConfig(
+                load_in_4bit=True,
+                bnb_4bit_use_double_quant=True,
+                bnb_4bit_quant_type="nf4",
+                bnb_4bit_compute_dtype=torch.bfloat16,
+            )
+            model = AutoModelForCausalLM.from_pretrained(
+                llm_name, quantization_config=bnb_config
+            )
+        else:
+            from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
+            tokenizer = AutoTokenizer.from_pretrained(llm_name)
+            model = AutoModelForCausalLM.from_pretrained(llm_name)
+
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
         pipe = pipeline(
-            "text-generation", model=model,
-            tokenizer=tokenizer, max_new_tokens=2000,
-            device_map="auto", eos_token_id=tokenizer.eos_token_id
+            "text-generation",
+            model=model,
+            tokenizer=tokenizer,
+            max_new_tokens=2000,
+            device_map="auto",
+            eos_token_id=tokenizer.eos_token_id,
         )
         hf = HuggingFacePipeline(pipeline=pipe)
 
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
 
         result = hf(prompt)
 
         return result.content
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
 
         raise NotImplementedError("HuggingFaceLLM Streaming is not implemented yet.")
-

application/llm/llama_cpp.py

@@ -1,39 +1,60 @@
 from application.llm.base import BaseLLM
 from application.core.settings import settings
+import threading
+
+
+class LlamaSingleton:
+    _instances = {}
+    _lock = threading.Lock()  # Add a lock for thread synchronization
+
+    @classmethod
+    def get_instance(cls, llm_name):
+        if llm_name not in cls._instances:
+            try:
+                from llama_cpp import Llama
+            except ImportError:
+                raise ImportError(
+                    "Please install llama_cpp using pip install llama-cpp-python"
+                )
+            cls._instances[llm_name] = Llama(model_path=llm_name, n_ctx=2048)
+        return cls._instances[llm_name]
+
+    @classmethod
+    def query_model(cls, llm, prompt, **kwargs):
+        with cls._lock:
+            return llm(prompt, **kwargs)
+
 
 class LlamaCpp(BaseLLM):
+    def __init__(
+        self,
+        api_key=None,
+        user_api_key=None,
+        llm_name=settings.LLM_PATH,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.llama = LlamaSingleton.get_instance(llm_name)
 
-    def __init__(self, api_key, llm_name=settings.MODEL_PATH, **kwargs):
-        global llama
-        try:
-            from llama_cpp import Llama
-        except ImportError:
-            raise ImportError("Please install llama_cpp using pip install llama-cpp-python")
-
-        llama = Llama(model_path=llm_name, n_ctx=2048)
-
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
+        result = LlamaSingleton.query_model(
+            self.llama, prompt, max_tokens=150, echo=False
+        )
+        return result["choices"][0]["text"].split("### Answer \n")[-1]
 
-        result = llama(prompt, max_tokens=150, echo=False)
-
-        # import sys
-        # print(result['choices'][0]['text'].split('### Answer \n')[-1], file=sys.stderr)
-        
-        return result['choices'][0]['text'].split('### Answer \n')[-1]
-
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
-        result = llama(prompt, max_tokens=150, echo=False, stream=stream)
-
-        # import sys
-        # print(list(result), file=sys.stderr)
-
+        result = LlamaSingleton.query_model(
+            self.llama, prompt, max_tokens=150, echo=False, stream=stream
+        )
         for item in result:
-            for choice in item['choices']:
-                yield choice['text']
+            for choice in item["choices"]:
+                yield choice["text"]

application/llm/llm_creator.py

@@ -1,22 +1,35 @@
+from application.llm.groq import GroqLLM
 from application.llm.openai import OpenAILLM, AzureOpenAILLM
 from application.llm.sagemaker import SagemakerAPILLM
 from application.llm.huggingface import HuggingFaceLLM
 from application.llm.llama_cpp import LlamaCpp
-
+from application.llm.anthropic import AnthropicLLM
+from application.llm.docsgpt_provider import DocsGPTAPILLM
+from application.llm.premai import PremAILLM
+from application.llm.google_ai import GoogleLLM
+from application.llm.novita import NovitaLLM
 
 
 class LLMCreator:
     llms = {
-        'openai': OpenAILLM,
-        'azure_openai': AzureOpenAILLM,
-        'sagemaker': SagemakerAPILLM,
-        'huggingface': HuggingFaceLLM,
-        'llama.cpp': LlamaCpp
+        "openai": OpenAILLM,
+        "azure_openai": AzureOpenAILLM,
+        "sagemaker": SagemakerAPILLM,
+        "huggingface": HuggingFaceLLM,
+        "llama.cpp": LlamaCpp,
+        "anthropic": AnthropicLLM,
+        "docsgpt": DocsGPTAPILLM,
+        "premai": PremAILLM,
+        "groq": GroqLLM,
+        "google": GoogleLLM,
+        "novita": NovitaLLM,
     }
 
     @classmethod
-    def create_llm(cls, type, *args, **kwargs):
+    def create_llm(cls, type, api_key, user_api_key, decoded_token, *args, **kwargs):
         llm_class = cls.llms.get(type.lower())
         if not llm_class:
             raise ValueError(f"No LLM class found for type {type}")
-        return llm_class(*args, **kwargs)
+        return llm_class(
+            api_key, user_api_key, decoded_token=decoded_token, *args, **kwargs
+        )

application/llm/novita.py

@@ -0,0 +1,32 @@
+from application.llm.base import BaseLLM
+from openai import OpenAI
+
+
+class NovitaLLM(BaseLLM):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.novita.ai/v3/openai")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs):
+        if tools:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, tools=tools, **kwargs
+            )
+            return response.choices[0]
+        else:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, **kwargs
+            )
+            return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self, baseself, model, messages, stream=True, tools=None, **kwargs
+    ):
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+        for line in response:
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content

application/llm/openai.py

@@ -1,57 +1,423 @@
-from application.llm.base import BaseLLM
+import base64
+import json
+import logging
+
 from application.core.settings import settings
+from application.llm.base import BaseLLM
+from application.storage.storage_creator import StorageCreator
+
 
 class OpenAILLM(BaseLLM):
 
-    def __init__(self, api_key):
-        global openai
-        import openai
-        openai.api_key = api_key
-        self.api_key = api_key  # Save the API key to be used later
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from openai import OpenAI
 
-    def _get_openai(self):
-        # Import openai when needed
-        import openai
-        # Set the API key every time you import openai
-        openai.api_key = self.api_key
-        return openai
+        super().__init__(*args, **kwargs)
+        if (
+            isinstance(settings.OPENAI_BASE_URL, str)
+            and settings.OPENAI_BASE_URL.strip()
+        ):
+            self.client = OpenAI(api_key=api_key, base_url=settings.OPENAI_BASE_URL)
+        else:
+            DEFAULT_OPENAI_API_BASE = "https://api.openai.com/v1"
+            self.client = OpenAI(api_key=api_key, base_url=DEFAULT_OPENAI_API_BASE)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.storage = StorageCreator.get_storage()
 
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        response = openai.ChatCompletion.create(
-            model=model,
-            engine=engine,
-            messages=messages,
-            stream=stream,
-            **kwargs
-        )
+    def _clean_messages_openai(self, messages):
+        cleaned_messages = []
+        for message in messages:
+            role = message.get("role")
+            content = message.get("content")
 
-        return response["choices"][0]["message"]["content"]
+            if role == "model":
+                role = "assistant"
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        response = openai.ChatCompletion.create(
-            model=model,
-            engine=engine,
-            messages=messages,
-            stream=stream,
-            **kwargs
-        )
+            if role and content is not None:
+                if isinstance(content, str):
+                    cleaned_messages.append({"role": role, "content": content})
+                elif isinstance(content, list):
+                    for item in content:
+                        if "text" in item:
+                            cleaned_messages.append(
+                                {"role": role, "content": item["text"]}
+                            )
+                        elif "function_call" in item:
+                            tool_call = {
+                                "id": item["function_call"]["call_id"],
+                                "type": "function",
+                                "function": {
+                                    "name": item["function_call"]["name"],
+                                    "arguments": json.dumps(
+                                        item["function_call"]["args"]
+                                    ),
+                                },
+                            }
+                            cleaned_messages.append(
+                                {
+                                    "role": "assistant",
+                                    "content": None,
+                                    "tool_calls": [tool_call],
+                                }
+                            )
+                        elif "function_response" in item:
+                            cleaned_messages.append(
+                                {
+                                    "role": "tool",
+                                    "tool_call_id": item["function_response"][
+                                        "call_id"
+                                    ],
+                                    "content": json.dumps(
+                                        item["function_response"]["response"]["result"]
+                                    ),
+                                }
+                            )
+                        elif isinstance(item, dict):
+                            content_parts = []
+                            if "text" in item:
+                                content_parts.append(
+                                    {"type": "text", "text": item["text"]}
+                                )
+                            elif (
+                                "type" in item
+                                and item["type"] == "text"
+                                and "text" in item
+                            ):
+                                content_parts.append(item)
+                            elif (
+                                "type" in item
+                                and item["type"] == "file"
+                                and "file" in item
+                            ):
+                                content_parts.append(item)
+                            elif (
+                                "type" in item
+                                and item["type"] == "image_url"
+                                and "image_url" in item
+                            ):
+                                content_parts.append(item)
+                            cleaned_messages.append(
+                                {"role": role, "content": content_parts}
+                            )
+                        else:
+                            raise ValueError(
+                                f"Unexpected content dictionary format: {item}"
+                            )
+                else:
+                    raise ValueError(f"Unexpected content type: {type(content)}")
+
+        return cleaned_messages
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        tools=None,
+        engine=settings.AZURE_DEPLOYMENT_NAME,
+        response_format=None,
+        **kwargs,
+    ):
+        messages = self._clean_messages_openai(messages)
+
+        request_params = {
+            "model": model,
+            "messages": messages,
+            "stream": stream,
+            **kwargs,
+        }
+
+        if tools:
+            request_params["tools"] = tools
+
+        if response_format:
+            request_params["response_format"] = response_format
+
+        response = self.client.chat.completions.create(**request_params)
+
+        if tools:
+            return response.choices[0]
+        else:
+            return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        tools=None,
+        engine=settings.AZURE_DEPLOYMENT_NAME,
+        response_format=None,
+        **kwargs,
+    ):
+        messages = self._clean_messages_openai(messages)
+
+        request_params = {
+            "model": model,
+            "messages": messages,
+            "stream": stream,
+            **kwargs,
+        }
+
+        if tools:
+            request_params["tools"] = tools
+
+        if response_format:
+            request_params["response_format"] = response_format
+
+        response = self.client.chat.completions.create(**request_params)
 
         for line in response:
-            if "content" in line["choices"][0]["delta"]:
-                yield line["choices"][0]["delta"]["content"]
+            if (
+                len(line.choices) > 0
+                and line.choices[0].delta.content is not None
+                and len(line.choices[0].delta.content) > 0
+            ):
+                yield line.choices[0].delta.content
+            elif len(line.choices) > 0:
+                yield line.choices[0]
+
+    def _supports_tools(self):
+        return True
+
+    def _supports_structured_output(self):
+        return True
+
+    def prepare_structured_output_format(self, json_schema):
+        if not json_schema:
+            return None
+
+        try:
+
+            def add_additional_properties_false(schema_obj):
+                if isinstance(schema_obj, dict):
+                    schema_copy = schema_obj.copy()
+
+                    if schema_copy.get("type") == "object":
+                        schema_copy["additionalProperties"] = False
+                        # Ensure 'required' includes all properties for OpenAI strict mode
+                        if "properties" in schema_copy:
+                            schema_copy["required"] = list(
+                                schema_copy["properties"].keys()
+                            )
+
+                    for key, value in schema_copy.items():
+                        if key == "properties" and isinstance(value, dict):
+                            schema_copy[key] = {
+                                prop_name: add_additional_properties_false(prop_schema)
+                                for prop_name, prop_schema in value.items()
+                            }
+                        elif key == "items" and isinstance(value, dict):
+                            schema_copy[key] = add_additional_properties_false(value)
+                        elif key in ["anyOf", "oneOf", "allOf"] and isinstance(
+                            value, list
+                        ):
+                            schema_copy[key] = [
+                                add_additional_properties_false(sub_schema)
+                                for sub_schema in value
+                            ]
+
+                    return schema_copy
+                return schema_obj
+
+            processed_schema = add_additional_properties_false(json_schema)
+
+            result = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": processed_schema.get("name", "response"),
+                    "description": processed_schema.get(
+                        "description", "Structured response"
+                    ),
+                    "schema": processed_schema,
+                    "strict": True,
+                },
+            }
+
+            return result
+
+        except Exception as e:
+            logging.error(f"Error preparing structured output format: {e}")
+            return None
+
+    def get_supported_attachment_types(self):
+        """
+        Return a list of MIME types supported by OpenAI for file uploads.
+
+        Returns:
+            list: List of supported MIME types
+        """
+        return [
+            "application/pdf",
+            "image/png",
+            "image/jpeg",
+            "image/jpg",
+            "image/webp",
+            "image/gif",
+        ]
+
+    def prepare_messages_with_attachments(self, messages, attachments=None):
+        """
+        Process attachments using OpenAI's file API for more efficient handling.
+
+        Args:
+            messages (list): List of message dictionaries.
+            attachments (list): List of attachment dictionaries with content and metadata.
+
+        Returns:
+            list: Messages formatted with file references for OpenAI API.
+        """
+        if not attachments:
+            return messages
+
+        prepared_messages = messages.copy()
+
+        # Find the user message to attach file_id to the last one
+        user_message_index = None
+        for i in range(len(prepared_messages) - 1, -1, -1):
+            if prepared_messages[i].get("role") == "user":
+                user_message_index = i
+                break
+
+        if user_message_index is None:
+            user_message = {"role": "user", "content": []}
+            prepared_messages.append(user_message)
+            user_message_index = len(prepared_messages) - 1
+
+        if isinstance(prepared_messages[user_message_index].get("content"), str):
+            text_content = prepared_messages[user_message_index]["content"]
+            prepared_messages[user_message_index]["content"] = [
+                {"type": "text", "text": text_content}
+            ]
+        elif not isinstance(prepared_messages[user_message_index].get("content"), list):
+            prepared_messages[user_message_index]["content"] = []
+
+        for attachment in attachments:
+            mime_type = attachment.get("mime_type")
+
+            if mime_type and mime_type.startswith("image/"):
+                try:
+                    base64_image = self._get_base64_image(attachment)
+                    prepared_messages[user_message_index]["content"].append(
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": f"data:{mime_type};base64,{base64_image}"
+                            },
+                        }
+                    )
+                except Exception as e:
+                    logging.error(
+                        f"Error processing image attachment: {e}", exc_info=True
+                    )
+                    if "content" in attachment:
+                        prepared_messages[user_message_index]["content"].append(
+                            {
+                                "type": "text",
+                                "text": f"[Image could not be processed: {attachment.get('path', 'unknown')}]",
+                            }
+                        )
+            # Handle PDFs using the file API
+            elif mime_type == "application/pdf":
+                try:
+                    file_id = self._upload_file_to_openai(attachment)
+                    prepared_messages[user_message_index]["content"].append(
+                        {"type": "file", "file": {"file_id": file_id}}
+                    )
+                except Exception as e:
+                    logging.error(f"Error uploading PDF to OpenAI: {e}", exc_info=True)
+                    if "content" in attachment:
+                        prepared_messages[user_message_index]["content"].append(
+                            {
+                                "type": "text",
+                                "text": f"File content:\n\n{attachment['content']}",
+                            }
+                        )
+
+        return prepared_messages
+
+    def _get_base64_image(self, attachment):
+        """
+        Convert an image file to base64 encoding.
+
+        Args:
+            attachment (dict): Attachment dictionary with path and metadata.
+
+        Returns:
+            str: Base64-encoded image data.
+        """
+        file_path = attachment.get("path")
+        if not file_path:
+            raise ValueError("No file path provided in attachment")
+
+        try:
+            with self.storage.get_file(file_path) as image_file:
+                return base64.b64encode(image_file.read()).decode("utf-8")
+        except FileNotFoundError:
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+    def _upload_file_to_openai(self, attachment):
+        """
+        Upload a file to OpenAI and return the file_id.
+
+        Args:
+            attachment (dict): Attachment dictionary with path and metadata.
+                Expected keys:
+                - path: Path to the file
+                - id: Optional MongoDB ID for caching
+
+        Returns:
+            str: OpenAI file_id for the uploaded file.
+        """
+        import logging
+
+        if "openai_file_id" in attachment:
+            return attachment["openai_file_id"]
+
+        file_path = attachment.get("path")
+
+        if not self.storage.file_exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        try:
+            file_id = self.storage.process_file(
+                file_path,
+                lambda local_path, **kwargs: self.client.files.create(
+                    file=open(local_path, "rb"), purpose="assistants"
+                ).id,
+            )
+
+            from application.core.mongo_db import MongoDB
+
+            mongo = MongoDB.get_client()
+            db = mongo[settings.MONGO_DB_NAME]
+            attachments_collection = db["attachments"]
+            if "_id" in attachment:
+                attachments_collection.update_one(
+                    {"_id": attachment["_id"]}, {"$set": {"openai_file_id": file_id}}
+                )
+
+            return file_id
+        except Exception as e:
+            logging.error(f"Error uploading file to OpenAI: {e}", exc_info=True)
+            raise
 
 
 class AzureOpenAILLM(OpenAILLM):
 
-    def __init__(self, openai_api_key, openai_api_base, openai_api_version, deployment_name):
-        super().__init__(openai_api_key)
-        self.api_base = settings.OPENAI_API_BASE,
-        self.api_version = settings.OPENAI_API_VERSION,
-        self.deployment_name = settings.AZURE_DEPLOYMENT_NAME,
+    def __init__(self, api_key, user_api_key, *args, **kwargs):
 
-    def _get_openai(self):
-        openai = super()._get_openai()
-        openai.api_base = self.api_base
-        openai.api_version = self.api_version
-        openai.api_type = "azure"
-        return openai
+        super().__init__(api_key)
+        self.api_base = (settings.OPENAI_API_BASE,)
+        self.api_version = (settings.OPENAI_API_VERSION,)
+        self.deployment_name = (settings.AZURE_DEPLOYMENT_NAME,)
+        from openai import AzureOpenAI
+
+        self.client = AzureOpenAI(
+            api_key=api_key,
+            api_version=settings.OPENAI_API_VERSION,
+            azure_endpoint=settings.OPENAI_API_BASE,
+        )

application/llm/premai.py

@@ -0,0 +1,38 @@
+from application.llm.base import BaseLLM
+from application.core.settings import settings
+
+
+class PremAILLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from premai import Prem
+
+        super().__init__(*args, **kwargs)
+        self.client = Prem(api_key=api_key)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.project_id = settings.PREMAI_PROJECT_ID
+
+    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+        response = self.client.chat.completions.create(
+            model=model,
+            project_id=self.project_id,
+            messages=messages,
+            stream=stream,
+            **kwargs
+        )
+
+        return response.choices[0].message["content"]
+
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+        response = self.client.chat.completions.create(
+            model=model,
+            project_id=self.project_id,
+            messages=messages,
+            stream=stream,
+            **kwargs
+        )
+
+        for line in response:
+            if line.choices[0].delta["content"] is not None:
+                yield line.choices[0].delta["content"]

application/llm/sagemaker.py

@@ -1,27 +1,140 @@
 from application.llm.base import BaseLLM
 from application.core.settings import settings
-import requests
 import json
+import io
+
+
+class LineIterator:
+    """
+    A helper class for parsing the byte stream input.
+
+    The output of the model will be in the following format:
+    ```
+    b'{"outputs": [" a"]}\n'
+    b'{"outputs": [" challenging"]}\n'
+    b'{"outputs": [" problem"]}\n'
+    ...
+    ```
+
+    While usually each PayloadPart event from the event stream will contain a byte array
+    with a full json, this is not guaranteed and some of the json objects may be split across
+    PayloadPart events. For example:
+    ```
+    {'PayloadPart': {'Bytes': b'{"outputs": '}}
+    {'PayloadPart': {'Bytes': b'[" problem"]}\n'}}
+    ```
+
+    This class accounts for this by concatenating bytes written via the 'write' function
+    and then exposing a method which will return lines (ending with a '\n' character) within
+    the buffer via the 'scan_lines' function. It maintains the position of the last read
+    position to ensure that previous bytes are not exposed again.
+    """
+
+    def __init__(self, stream):
+        self.byte_iterator = iter(stream)
+        self.buffer = io.BytesIO()
+        self.read_pos = 0
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        while True:
+            self.buffer.seek(self.read_pos)
+            line = self.buffer.readline()
+            if line and line[-1] == ord("\n"):
+                self.read_pos += len(line)
+                return line[:-1]
+            try:
+                chunk = next(self.byte_iterator)
+            except StopIteration:
+                if self.read_pos < self.buffer.getbuffer().nbytes:
+                    continue
+                raise
+            if "PayloadPart" not in chunk:
+                print("Unknown event type:" + chunk)
+                continue
+            self.buffer.seek(0, io.SEEK_END)
+            self.buffer.write(chunk["PayloadPart"]["Bytes"])
+
 
 class SagemakerAPILLM(BaseLLM):
 
-    def __init__(self, *args, **kwargs):
-        self.url = settings.SAGEMAKER_API_URL
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        import boto3
 
-    def gen(self, model, engine, messages, stream=False, **kwargs):
-        context = messages[0]['content']
-        user_question = messages[-1]['content']
-        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
-        response = requests.post(
-                    url=self.url,
-                    headers={
-                        "Content-Type": "application/json; charset=utf-8",
-                    },
-                    data=json.dumps({"input": prompt})
+        runtime = boto3.client(
+            "runtime.sagemaker",
+            aws_access_key_id="xxx",
+            aws_secret_access_key="xxx",
+            region_name="us-west-2",
         )
 
-        return response.json()['answer']
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.endpoint = settings.SAGEMAKER_ENDPOINT
+        self.runtime = runtime
 
-    def gen_stream(self, model, engine, messages, stream=True, **kwargs):
-        raise NotImplementedError("Sagemaker does not support streaming")
+    def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
+        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
+
+        # Construct payload for endpoint
+        payload = {
+            "inputs": prompt,
+            "stream": False,
+            "parameters": {
+                "do_sample": True,
+                "temperature": 0.1,
+                "max_new_tokens": 30,
+                "repetition_penalty": 1.03,
+                "stop": ["</s>", "###"],
+            },
+        }
+        body_bytes = json.dumps(payload).encode("utf-8")
+
+        # Invoke the endpoint
+        response = self.runtime.invoke_endpoint(
+            EndpointName=self.endpoint, ContentType="application/json", Body=body_bytes
+        )
+        result = json.loads(response["Body"].read().decode())
+        import sys
+
+        print(result[0]["generated_text"], file=sys.stderr)
+        return result[0]["generated_text"][len(prompt) :]
+
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, tools=None, **kwargs):
+        context = messages[0]["content"]
+        user_question = messages[-1]["content"]
+        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
+
+        # Construct payload for endpoint
+        payload = {
+            "inputs": prompt,
+            "stream": True,
+            "parameters": {
+                "do_sample": True,
+                "temperature": 0.1,
+                "max_new_tokens": 512,
+                "repetition_penalty": 1.03,
+                "stop": ["</s>", "###"],
+            },
+        }
+        body_bytes = json.dumps(payload).encode("utf-8")
+
+        # Invoke the endpoint
+        response = self.runtime.invoke_endpoint_with_response_stream(
+            EndpointName=self.endpoint, ContentType="application/json", Body=body_bytes
+        )
+        # result = json.loads(response['Body'].read().decode())
+        event_stream = response["Body"]
+        start_json = b"{"
+        for line in LineIterator(event_stream):
+            if line != b"" and start_json in line:
+                # print(line)
+                data = json.loads(line[line.find(start_json) :].decode("utf-8"))
+                if data["token"]["text"] not in ["</s>", "###"]:
+                    print(data["token"]["text"], end="")
+                    yield data["token"]["text"]

application/logging.py

@@ -0,0 +1,161 @@
+import datetime
+import functools
+import inspect
+
+import logging
+import uuid
+from typing import Any, Callable, Dict, Generator, List
+
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+
+
+class LogContext:
+    def __init__(self, endpoint, activity_id, user, api_key, query):
+        self.endpoint = endpoint
+        self.activity_id = activity_id
+        self.user = user
+        self.api_key = api_key
+        self.query = query
+        self.stacks = []
+
+
+def build_stack_data(
+    obj: Any,
+    include_attributes: List[str] = None,
+    exclude_attributes: List[str] = None,
+    custom_data: Dict = None,
+) -> Dict:
+    if obj is None:
+        raise ValueError("The 'obj' parameter cannot be None")
+    data = {}
+    if include_attributes is None:
+        include_attributes = []
+        for name, value in inspect.getmembers(obj):
+            if (
+                not name.startswith("_")
+                and not inspect.ismethod(value)
+                and not inspect.isfunction(value)
+            ):
+                include_attributes.append(name)
+    for attr_name in include_attributes:
+        if exclude_attributes and attr_name in exclude_attributes:
+            continue
+        try:
+            attr_value = getattr(obj, attr_name)
+            if attr_value is not None:
+                if isinstance(attr_value, (int, float, str, bool)):
+                    data[attr_name] = attr_value
+                elif isinstance(attr_value, list):
+                    if all(isinstance(item, dict) for item in attr_value):
+                        data[attr_name] = attr_value
+                    elif all(hasattr(item, "__dict__") for item in attr_value):
+                        data[attr_name] = [item.__dict__ for item in attr_value]
+                    else:
+                        data[attr_name] = [str(item) for item in attr_value]
+                elif isinstance(attr_value, dict):
+                    data[attr_name] = {k: str(v) for k, v in attr_value.items()}
+        except AttributeError as e:
+            logging.warning(f"AttributeError while accessing {attr_name}: {e}")
+        except AttributeError:
+            pass
+    if custom_data:
+        data.update(custom_data)
+    return data
+
+
+def log_activity() -> Callable:
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            activity_id = str(uuid.uuid4())
+            data = build_stack_data(args[0])
+            endpoint = data.get("endpoint", "")
+            user = data.get("user", "local")
+            api_key = data.get("user_api_key", "")
+            query = kwargs.get("query", getattr(args[0], "query", ""))
+
+            context = LogContext(endpoint, activity_id, user, api_key, query)
+            kwargs["log_context"] = context
+
+            logging.info(
+                f"Starting activity: {endpoint} - {activity_id} - User: {user}"
+            )
+
+            generator = func(*args, **kwargs)
+            yield from _consume_and_log(generator, context)
+
+        return wrapper
+
+    return decorator
+
+
+def _consume_and_log(generator: Generator, context: "LogContext"):
+    try:
+        for item in generator:
+            yield item
+    except Exception as e:
+        logging.exception(f"Error in {context.endpoint} - {context.activity_id}: {e}")
+        context.stacks.append({"component": "error", "data": {"message": str(e)}})
+        _log_to_mongodb(
+            endpoint=context.endpoint,
+            activity_id=context.activity_id,
+            user=context.user,
+            api_key=context.api_key,
+            query=context.query,
+            stacks=context.stacks,
+            level="error",
+        )
+        raise
+    finally:
+        _log_to_mongodb(
+            endpoint=context.endpoint,
+            activity_id=context.activity_id,
+            user=context.user,
+            api_key=context.api_key,
+            query=context.query,
+            stacks=context.stacks,
+            level="info",
+        )
+
+
+def _log_to_mongodb(
+    endpoint: str,
+    activity_id: str,
+    user: str,
+    api_key: str,
+    query: str,
+    stacks: List[Dict],
+    level: str,
+) -> None:
+    try:
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        user_logs_collection = db["stack_logs"]
+        
+
+
+        log_entry = {
+            "endpoint": endpoint,
+            "id": activity_id,
+            "level": level,
+            "user": user,
+            "api_key": api_key,
+            "query": query,
+            "stacks": stacks,
+            "timestamp": datetime.datetime.now(datetime.timezone.utc),
+        }
+        # clean up text fields to be no longer than 10000 characters
+        for key, value in log_entry.items():
+            if isinstance(value, str) and len(value) > 10000:
+                log_entry[key] = value[:10000]
+    
+        user_logs_collection.insert_one(log_entry)
+        logging.debug(f"Logged activity to MongoDB: {activity_id}")
+
+    except Exception as e:
+        logging.error(f"Failed to log to MongoDB: {e}", exc_info=True)

application/package.json

@@ -1,5 +0,0 @@
-{
-  "devDependencies": {
-    "tailwindcss": "^3.2.4"
-  }
-}

application/parser/chunking.py

@@ -0,0 +1,94 @@
+import re
+from typing import List, Tuple
+import logging
+from application.parser.schema.base import Document
+from application.utils import get_encoding
+
+logger = logging.getLogger(__name__)
+
+class Chunker:
+    def __init__(
+        self,
+        chunking_strategy: str = "classic_chunk",
+        max_tokens: int = 2000,
+        min_tokens: int = 150,
+        duplicate_headers: bool = False,
+    ):
+        if chunking_strategy not in ["classic_chunk"]:
+            raise ValueError(f"Unsupported chunking strategy: {chunking_strategy}")
+        self.chunking_strategy = chunking_strategy
+        self.max_tokens = max_tokens
+        self.min_tokens = min_tokens
+        self.duplicate_headers = duplicate_headers
+        self.encoding = get_encoding()
+
+    def separate_header_and_body(self, text: str) -> Tuple[str, str]:
+        header_pattern = r"^(.*?\n){3}"
+        match = re.match(header_pattern, text)
+        if match:
+            header = match.group(0)
+            body = text[len(header):]
+        else:
+            header, body = "", text  # No header, treat entire text as body
+        return header, body
+
+
+    
+    def split_document(self, doc: Document) -> List[Document]:
+        split_docs = []
+        header, body = self.separate_header_and_body(doc.text)
+        header_tokens = self.encoding.encode(header) if header else []
+        body_tokens = self.encoding.encode(body)
+
+        current_position = 0
+        part_index = 0
+        while current_position < len(body_tokens):
+            end_position = current_position + self.max_tokens - len(header_tokens)
+            chunk_tokens = (header_tokens + body_tokens[current_position:end_position]
+                            if self.duplicate_headers or part_index == 0 else body_tokens[current_position:end_position])
+            chunk_text = self.encoding.decode(chunk_tokens)
+            new_doc = Document(
+                text=chunk_text,
+                doc_id=f"{doc.doc_id}-{part_index}",
+                embedding=doc.embedding,
+                extra_info={**(doc.extra_info or {}), "token_count": len(chunk_tokens)}
+            )
+            split_docs.append(new_doc)
+            current_position = end_position
+            part_index += 1
+            header_tokens = []
+        return split_docs
+
+    def classic_chunk(self, documents: List[Document]) -> List[Document]:
+        processed_docs = []
+        i = 0
+        while i < len(documents):
+            doc = documents[i]
+            tokens = self.encoding.encode(doc.text)
+            token_count = len(tokens)
+
+            if self.min_tokens <= token_count <= self.max_tokens:
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            elif token_count < self.min_tokens:
+  
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            else:
+                # Split large documents
+                processed_docs.extend(self.split_document(doc))
+                i += 1
+        return processed_docs
+
+    def chunk(
+        self,
+        documents: List[Document]
+    ) -> List[Document]:
+        if self.chunking_strategy == "classic_chunk":
+            return self.classic_chunk(documents)
+        else:
+            raise ValueError("Unsupported chunking strategy")

application/parser/connectors/__init__.py

@@ -0,0 +1,18 @@
+"""
+External knowledge base connectors for DocsGPT.
+
+This module contains connectors for external knowledge bases and document storage systems
+that require authentication and specialized handling, separate from simple web scrapers.
+"""
+
+from .base import BaseConnectorAuth, BaseConnectorLoader
+from .connector_creator import ConnectorCreator
+from .google_drive import GoogleDriveAuth, GoogleDriveLoader
+
+__all__ = [
+    'BaseConnectorAuth',
+    'BaseConnectorLoader',
+    'ConnectorCreator',
+    'GoogleDriveAuth',
+    'GoogleDriveLoader'
+]

application/parser/connectors/base.py

@@ -0,0 +1,129 @@
+"""
+Base classes for external knowledge base connectors.
+
+This module provides minimal abstract base classes that define the essential
+interface for external knowledge base connectors.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from application.parser.schema.base import Document
+
+
+class BaseConnectorAuth(ABC):
+    """
+    Abstract base class for connector authentication.
+    
+    Defines the minimal interface that all connector authentication
+    implementations must follow.
+    """
+    
+    @abstractmethod
+    def get_authorization_url(self, state: Optional[str] = None) -> str:
+        """
+        Generate authorization URL for OAuth flows.
+        
+        Args:
+            state: Optional state parameter for CSRF protection
+            
+        Returns:
+            Authorization URL
+        """
+        pass
+    
+    @abstractmethod
+    def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:
+        """
+        Exchange authorization code for access tokens.
+        
+        Args:
+            authorization_code: Authorization code from OAuth callback
+            
+        Returns:
+            Dictionary containing token information
+        """
+        pass
+    
+    @abstractmethod
+    def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
+        """
+        Refresh an expired access token.
+        
+        Args:
+            refresh_token: Refresh token
+            
+        Returns:
+            Dictionary containing refreshed token information
+        """
+        pass
+    
+    @abstractmethod
+    def is_token_expired(self, token_info: Dict[str, Any]) -> bool:
+        """
+        Check if a token is expired.
+        
+        Args:
+            token_info: Token information dictionary
+            
+        Returns:
+            True if token is expired, False otherwise
+        """
+        pass
+
+
+class BaseConnectorLoader(ABC):
+    """
+    Abstract base class for connector loaders.
+    
+    Defines the minimal interface that all connector loader
+    implementations must follow.
+    """
+    
+    @abstractmethod
+    def __init__(self, session_token: str):
+        """
+        Initialize the connector loader.
+        
+        Args:
+            session_token: Authentication session token
+        """
+        pass
+    
+    @abstractmethod
+    def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
+        """
+        Load documents from the external knowledge base.
+        
+        Args:
+            inputs: Configuration dictionary containing:
+                - file_ids: Optional list of specific file IDs to load
+                - folder_ids: Optional list of folder IDs to browse/download
+                - limit: Maximum number of items to return
+                - list_only: If True, return metadata without content
+                - recursive: Whether to recursively process folders
+                
+        Returns:
+            List of Document objects
+        """
+        pass
+    
+    @abstractmethod
+    def download_to_directory(self, local_dir: str, source_config: Dict[str, Any] = None) -> Dict[str, Any]:
+        """
+        Download files/folders to a local directory.
+        
+        Args:
+            local_dir: Local directory path to download files to
+            source_config: Configuration for what to download
+            
+        Returns:
+            Dictionary containing download results:
+                - files_downloaded: Number of files downloaded
+                - directory_path: Path where files were downloaded
+                - empty_result: Whether no files were downloaded
+                - source_type: Type of connector
+                - config_used: Configuration that was used
+                - error: Error message if download failed (optional)
+        """
+        pass

application/parser/connectors/connector_creator.py

@@ -0,0 +1,81 @@
+from application.parser.connectors.google_drive.loader import GoogleDriveLoader
+from application.parser.connectors.google_drive.auth import GoogleDriveAuth
+
+
+class ConnectorCreator:
+    """
+    Factory class for creating external knowledge base connectors and auth providers.
+
+    These are different from remote loaders as they typically require
+    authentication and connect to external document storage systems.
+    """
+
+    connectors = {
+        "google_drive": GoogleDriveLoader,
+    }
+
+    auth_providers = {
+        "google_drive": GoogleDriveAuth,
+    }
+
+    @classmethod
+    def create_connector(cls, connector_type, *args, **kwargs):
+        """
+        Create a connector instance for the specified type.
+
+        Args:
+            connector_type: Type of connector to create (e.g., 'google_drive')
+            *args, **kwargs: Arguments to pass to the connector constructor
+
+        Returns:
+            Connector instance
+
+        Raises:
+            ValueError: If connector type is not supported
+        """
+        connector_class = cls.connectors.get(connector_type.lower())
+        if not connector_class:
+            raise ValueError(f"No connector class found for type {connector_type}")
+        return connector_class(*args, **kwargs)
+
+    @classmethod
+    def create_auth(cls, connector_type):
+        """
+        Create an auth provider instance for the specified connector type.
+
+        Args:
+            connector_type: Type of connector auth to create (e.g., 'google_drive')
+
+        Returns:
+            Auth provider instance
+
+        Raises:
+            ValueError: If connector type is not supported for auth
+        """
+        auth_class = cls.auth_providers.get(connector_type.lower())
+        if not auth_class:
+            raise ValueError(f"No auth class found for type {connector_type}")
+        return auth_class()
+
+    @classmethod
+    def get_supported_connectors(cls):
+        """
+        Get list of supported connector types.
+
+        Returns:
+            List of supported connector type strings
+        """
+        return list(cls.connectors.keys())
+
+    @classmethod
+    def is_supported(cls, connector_type):
+        """
+        Check if a connector type is supported.
+
+        Args:
+            connector_type: Type of connector to check
+
+        Returns:
+            True if supported, False otherwise
+        """
+        return connector_type.lower() in cls.connectors

application/parser/connectors/google_drive/__init__.py

@@ -0,0 +1,10 @@
+"""
+Google Drive connector for DocsGPT.
+
+This module provides authentication and document loading capabilities for Google Drive.
+"""
+
+from .auth import GoogleDriveAuth
+from .loader import GoogleDriveLoader
+
+__all__ = ['GoogleDriveAuth', 'GoogleDriveLoader']