fix(frontend): use bracket notation for tool variable paths (#2176)

Adds complete German (Deutsch) language support to DocsGPT.

Changes:
- Add de.json with full German translations
- Register German in i18n configuration
- Add German to language selector dropdown

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude <noreply@anthropic.com>

.devcontainer/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.12-bookworm
+
+# Install Node.js 20.x
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install global npm packages
+RUN npm install -g husky vite
+
+# Create and activate Python virtual environment
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+WORKDIR /workspace

.devcontainer/devc-welcome.md

@@ -0,0 +1,49 @@
+# Welcome to DocsGPT Devcontainer
+
+Welcome to the DocsGPT development environment! This guide will help you get started quickly.
+
+## Starting Services
+
+To run DocsGPT, you need to start three main services: Flask (backend), Celery (task queue), and Vite (frontend). Here are the commands to start each service within the devcontainer:
+
+### Vite (Frontend)
+
+```bash
+cd frontend
+npm run dev -- --host
+```
+
+### Flask (Backend)
+
+```bash
+flask --app application/app.py run --host=0.0.0.0 --port=7091
+```
+
+### Celery (Task Queue)
+
+```bash
+celery -A application.app.celery worker -l INFO
+```
+
+## Github Codespaces Instructions
+
+### 1. Make Ports Public:
+
+Go to the "Ports" panel in Codespaces (usually located at the bottom of the VS Code window).
+
+For both port 5173 and 7091, right-click on the port and select "Make Public".
+
+![CleanShot 2025-02-12 at 09 46 14@2x](https://github.com/user-attachments/assets/00a34b16-a7ef-47af-9648-87a7e3008475)
+
+
+ ### 2. Update VITE_API_HOST:
+
+After making port 7091 public, copy the public URL provided by Codespaces for port 7091.
+
+Open the file frontend/.env.development.
+
+Find the line VITE_API_HOST=http://localhost:7091.
+
+Replace http://localhost:7091 with the public URL you copied from Codespaces.
+
+![CleanShot 2025-02-12 at 09 46 56@2x](https://github.com/user-attachments/assets/c472242f-1079-4cd8-bc0b-2d78db22b94c)

.devcontainer/devcontainer.json

@@ -0,0 +1,24 @@
+{
+	"name": "DocsGPT Dev Container",
+	"dockerComposeFile": ["docker-compose-dev.yaml", "docker-compose.override.yaml"],
+	"service": "dev",
+	"workspaceFolder": "/workspace",
+	"postCreateCommand": ".devcontainer/post-create-command.sh",
+	"forwardPorts": [7091, 5173, 6379, 27017],
+	"customizations": {
+	  "vscode": {
+		"extensions": [
+		  "ms-python.python",
+		  "ms-toolsai.jupyter",
+		  "esbenp.prettier-vscode",
+		  "dbaeumer.vscode-eslint"
+		]
+	  },
+	  "codespaces": {
+			"openFiles": [
+			".devcontainer/devc-welcome.md",
+			"CONTRIBUTING.md"
+			]
+		}
+	}
+  }

.devcontainer/docker-compose-dev.yaml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
 
   redis:

.devcontainer/docker-compose.override.yaml

@@ -0,0 +1,40 @@
+version: '3.8'
+
+services:
+  dev:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ../:/workspace:cached
+    command: sleep infinity
+    depends_on:
+      redis:
+        condition: service_healthy
+      mongo:
+        condition: service_healthy
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/1
+      - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - CACHE_REDIS_URL=redis://redis:6379/2
+    networks:
+      - default
+
+  redis:
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 30s
+      retries: 5
+
+  mongo:
+    healthcheck:
+      test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
+      interval: 5s
+      timeout: 30s
+      retries: 5
+
+networks:
+  default:
+    name: docsgpt-dev-network

.devcontainer/post-create-command.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e  # Exit immediately if a command exits with a non-zero status
+
+if [ ! -f frontend/.env.development ]; then
+  cp -n .env-template frontend/.env.development || true # Assuming .env-template is in the root
+fi
+
+# Determine VITE_API_HOST based on environment
+if [ -n "$CODESPACES" ]; then
+  # Running in Codespaces
+  CODESPACE_NAME=$(echo "$CODESPACES" | cut -d'-' -f1) # Extract codespace name
+  PUBLIC_API_HOST="https://${CODESPACE_NAME}-7091.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN}"
+  echo "Setting VITE_API_HOST for Codespaces: $PUBLIC_API_HOST in frontend/.env.development"
+  sed -i "s|VITE_API_HOST=.*|VITE_API_HOST=$PUBLIC_API_HOST|" frontend/.env.development
+else
+  # Not running in Codespaces (local devcontainer)
+  DEFAULT_API_HOST="http://localhost:7091"
+  echo "Setting VITE_API_HOST for local dev: $DEFAULT_API_HOST in frontend/.env.development"
+  sed -i "s|VITE_API_HOST=.*|VITE_API_HOST=$DEFAULT_API_HOST|" frontend/.env.development
+fi
+
+
+mkdir -p model
+if [ ! -d model/all-mpnet-base-v2 ]; then
+    wget -q https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip -O model/mpnet-base-v2.zip
+    unzip -q model/mpnet-base-v2.zip -d model
+    rm model/mpnet-base-v2.zip
+fi
+pip install -r application/requirements.txt
+cd frontend
+npm install --include=dev

.env-template

@@ -1,2 +1,9 @@
-OPENAI_API_KEY=<LLM api key (for example, open ai key)>
-EMBEDDINGS_KEY=<LLM embeddings api key (for example, open ai key)>
+API_KEY=<LLM api key (for example, open ai key)>
+LLM_NAME=docsgpt
+VITE_API_STREAMING=true
+
+#For Azure (you can delete it if you don't use Azure)
+OPENAI_API_BASE=
+OPENAI_API_VERSION=
+AZURE_DEPLOYMENT_NAME=
+AZURE_EMBEDDINGS_DEPLOYMENT_NAME=

.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: arc53

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,138 @@
+name: "🐛 Bug Report"
+description: "Submit a bug report to help us improve"
+title: "🐛 Bug Report: "
+labels: ["type: bug"]
+body:
+  - type: markdown
+    attributes:
+      value: We value your time and your efforts to submit this bug report is appreciated. 🙏
+
+  - type: textarea
+    id: description
+    validations:
+      required: true
+    attributes:
+      label: "📜 Description"
+      description: "A clear and concise description of what the bug is."
+      placeholder: "It bugs out when ..."
+
+  - type: textarea
+    id: steps-to-reproduce
+    validations:
+      required: true
+    attributes:
+      label: "👟 Reproduction steps"
+      description: "How do you trigger this bug? Please walk us through it step by step."
+      placeholder: "1. Go to '...'
+        2. Click on '....'
+        3. Scroll down to '....'
+        4. See error"
+
+  - type: textarea
+    id: expected-behavior
+    validations:
+      required: true
+    attributes:
+      label: "👍 Expected behavior"
+      description: "What did you think should happen?"
+      placeholder: "It should ..."
+
+  - type: textarea
+    id: actual-behavior
+    validations:
+      required: true
+    attributes:
+      label: "👎 Actual Behavior with Screenshots"
+      description: "What did actually happen? Add screenshots, if applicable."
+      placeholder: "It actually ..."
+
+  - type: dropdown
+    id: operating-system
+    attributes:
+      label: "💻 Operating system"
+      description: "What OS is your app running on?"
+      options:
+        - Linux
+        - MacOS
+        - Windows
+        - Something else
+    validations:
+      required: true
+
+  - type: dropdown
+    id: browsers
+    attributes:
+      label: What browsers are you seeing the problem on?
+      multiple: true
+      options:
+        - Firefox
+        - Chrome
+        - Safari
+        - Microsoft Edge
+        - Something else
+
+  - type: dropdown
+    id: dev-environment
+    validations:
+      required: true
+    attributes:
+      label: "🤖 What development environment are you experiencing this bug on?"
+      options:
+        - Docker
+        - Local dev server
+
+  - type: textarea
+    id: env-vars
+    validations:
+      required: false
+    attributes:
+      label: "🔒 Did you set the correct environment variables in the right path? List the environment variable names (not values please!)"
+      description: "Please refer to the [Project setup instructions](https://github.com/arc53/DocsGPT#quickstart) if you are unsure."
+      placeholder: "It actually ..."
+
+  - type: textarea
+    id: additional-context
+    validations:
+      required: false
+    attributes:
+      label: "📃 Provide any additional context for the Bug."
+      description: "Add any other context about the problem here."
+      placeholder: "It actually ..."
+
+  - type: textarea
+    id: logs
+    validations:
+      required: false
+    attributes:
+      label: 📖 Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: shell
+
+  - type: checkboxes
+    id: no-duplicate-issues
+    attributes:
+      label: "👀 Have you spent some time to check if this bug has been raised before?"
+      options:
+        - label: "I checked and didn't find similar issue"
+          required: true
+
+  - type: dropdown
+    id: willing-to-submit-pr
+    attributes:
+      label: 🔗 Are you willing to submit PR?
+      description: This is absolutely not required, but we are happy to guide you in the contribution process.
+      options: # Added options key
+        - "Yes, I am willing to submit a PR!"
+        - "No"
+    validations:
+      required: false
+
+
+  - type: checkboxes
+    id: terms
+    attributes:
+      label: 🧑‍⚖️ Code of Conduct
+      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/arc53/DocsGPT/blob/main/CODE_OF_CONDUCT.md)
+      options:
+        - label: I agree to follow this project's Code of Conduct
+          required: true

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,54 @@
+name: 🚀 Feature
+description: "Submit a proposal for a new feature"
+title: "🚀 Feature: "
+labels: [feature]
+body:
+  - type: markdown
+    attributes:
+      value: We value your time and your efforts to submit this bug report is appreciated. 🙏
+  - type: textarea
+    id: feature-description
+    validations:
+      required: true
+    attributes:
+      label: "🔖 Feature description"
+      description: "A clear and concise description of what the feature is."
+      placeholder: "You should add ..."
+  - type: textarea
+    id: pitch
+    validations:
+      required: true
+    attributes:
+      label: "🎤 Why is this feature needed ?"
+      description: "Please explain why this feature should be implemented and how it would be used. Add examples, if applicable."
+      placeholder: "In my use-case, ..."
+  - type: textarea
+    id: solution
+    validations:
+      required: true
+    attributes:
+      label: "✌️ How do you aim to achieve this?"
+      description: "A clear and concise description of what you want to happen."
+      placeholder: "I want this feature to, ..."
+  - type: textarea
+    id: alternative
+    validations:
+      required: false
+    attributes:
+      label: "🔄️ Additional Information"
+      description: "A clear and concise description of any alternative solutions or additional solutions you've considered."
+      placeholder: "I tried, ..."
+  - type: checkboxes
+    id: no-duplicate-issues
+    attributes:
+      label: "👀 Have you spent some time to check if this feature request has been raised before?"
+      options:
+        - label: "I checked and didn't find similar issue"
+          required: true
+  - type: dropdown
+    id: willing-to-submit-pr
+    attributes:
+      label: Are you willing to submit PR?
+      description: This is absolutely not required, but we are happy to guide you in the contribution process.
+      options:
+        - "Yes I am willing to submit a PR!"

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,5 @@
+- **What kind of change does this PR introduce?** (Bug fix, feature, docs update, ...)
+
+- **Why was this change needed?** (You can also link to an open issue here)
+
+- **Other information**:

.github/dependabot.yml

@@ -0,0 +1,23 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/application" # Location of package manifests
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm" # See documentation for possible values
+    directory: "/frontend" # Location of package manifests
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "npm"
+    directory: "/extensions/react-widget"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/holopin.yml

@@ -0,0 +1,11 @@
+organization: docsgpt
+defaultSticker: cm1ulwkkl180570cl82rtzympu
+stickers:
+  - id: cm1ulwkkl180570cl82rtzympu
+    alias: contributor2024
+  - id: cm1ureg8o130450cl8c1po6mil
+    alias: api
+  - id: cm1urhmag148240cl8yvqxkthx
+    alias: lpc
+  - id: cm1urlcpq622090cl2tvu4w71y
+    alias: lexeu

.github/labeler.yml

@@ -0,0 +1,31 @@
+repo:
+- changed-files:
+  - any-glob-to-any-file: '*'
+
+github:
+- changed-files:
+  - any-glob-to-any-file: '.github/**/*'
+
+application:
+- changed-files:
+  - any-glob-to-any-file: 'application/**/*'
+
+docs:
+- changed-files:
+  - any-glob-to-any-file: 'docs/**/*'
+
+extensions:
+- changed-files:
+  - any-glob-to-any-file: 'extensions/**/*'
+
+frontend:
+- changed-files:
+  - any-glob-to-any-file: 'frontend/**/*'
+
+scripts:
+- changed-files:
+  - any-glob-to-any-file: 'scripts/**/*'
+
+tests:
+- changed-files:
+  - any-glob-to-any-file: 'tests/**/*'

.github/styles/DocsGPT/Spelling.yml

@@ -0,0 +1,11 @@
+extends: spelling
+level: warning
+message: "Did you really mean '%s'?"
+ignore:
+  - "**/node_modules/**"
+  - "**/dist/**"
+  - "**/build/**"
+  - "**/coverage/**"
+  - "**/public/**"
+  - "**/static/**"
+vocab: DocsGPT

.github/styles/config/vocabularies/DocsGPT/accept.txt

@@ -0,0 +1,46 @@
+Ollama
+Qdrant
+Milvus
+Chatwoot
+Nextra
+VSCode
+npm
+LLMs
+APIs
+Groq
+SGLang
+LMDeploy
+OAuth
+Vite
+LLM
+JSONPath
+UIs
+configs
+uncomment
+qdrant
+vectorstore
+docsgpt
+llm
+GPUs
+kubectl
+Lightsail
+enqueues
+chatbot
+VSCode's
+Shareability
+feedbacks
+automations
+Premade
+Signup
+Repo
+repo
+env
+URl
+agentic
+llama_cpp
+parsable
+SDKs
+boolean
+bool
+hardcode
+EOL

.github/workflows/bandit.yaml

@@ -0,0 +1,40 @@
+name: Bandit Security Scan
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  bandit_scan:
+    if: ${{ github.repository == 'arc53/DocsGPT' }}
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      actions: read
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install bandit  # Bandit is needed for this action
+          if [ -f application/requirements.txt ]; then pip install -r application/requirements.txt; fi
+
+      - name: Run Bandit scan
+        uses: PyCQA/bandit-action@v1
+        with:
+          severity: medium
+          confidence: medium
+          targets: application/
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -1,48 +1,112 @@
 name: Build and push DocsGPT Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
-  deploy:
-    runs-on: ubuntu-latest
+  build:
+    if: github.repository == 'arc53/DocsGPT'
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            suffix: amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            suffix: arm64
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
-
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+      - name: Set up QEMU  # Only needed for emulation, not for native arm64 builds
+        if: matrix.platform == 'linux/arm64'
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Runs a single command using the runners shell
-      - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+      - name: Build and push platform-specific images
+        uses: docker/build-push-action@v6
         with:
           file: './application/Dockerfile'
-          platforms: linux/amd64
+          platforms: ${{ matrix.platform }}
           context: ./application
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
+            ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
+          provenance: false
+          sbom: false
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+          cache-to: type=inline
+
+  manifest:
+    if: github.repository == 'arc53/DocsGPT'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest for DockerHub
+        run: |
+          set -e
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }} \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+
+      - name: Create and push manifest for ghcr.io
+        run: |
+          set -e
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }} \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:latest \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:latest

.github/workflows/cife.yml

@@ -1,48 +1,112 @@
 name: Build and push DocsGPT-FE Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
-  deploy:
-    runs-on: ubuntu-latest
+  build:
+    if: github.repository == 'arc53/DocsGPT'
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            suffix: amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            suffix: arm64
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
-
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+      - name: Set up QEMU # Only needed for emulation, not for native arm64 builds
+        if: matrix.platform == 'linux/arm64'
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Runs a single command using the runners shell
-      - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+      - name: Build and push platform-specific images
+        uses: docker/build-push-action@v6
         with:
           file: './frontend/Dockerfile'
-          platforms: linux/amd64
+          platforms: ${{ matrix.platform }}
           context: ./frontend
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-${{ matrix.suffix }}
+          provenance: false
+          sbom: false
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+          cache-to: type=inline
+
+  manifest:
+    if: github.repository == 'arc53/DocsGPT'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest for DockerHub
+        run: |
+          set -e
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }} \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+
+      - name: Create and push manifest for ghcr.io
+        run: |
+          set -e
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }} \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest

.github/workflows/docker-develop-build.yml

@@ -0,0 +1,100 @@
+name: Build and push multi-arch DocsGPT Docker image
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    if: github.repository == 'arc53/DocsGPT'
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            suffix: amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            suffix: arm64
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push platform-specific images
+        uses: docker/build-push-action@v6
+        with:
+          file: './application/Dockerfile'
+          platforms: ${{ matrix.platform }}
+          context: ./application
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-${{ matrix.suffix }}
+            ghcr.io/${{ github.repository_owner }}/docsgpt:develop-${{ matrix.suffix }}
+          provenance: false
+          sbom: false
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+          cache-to: type=inline
+
+  manifest:
+    if: github.repository == 'arc53/DocsGPT'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+        
+      - name: Create and push manifest for DockerHub
+        run: |
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+
+      - name: Create and push manifest for ghcr.io
+        run: |
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:develop \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:develop

.github/workflows/docker-develop-fe-build.yml

@@ -0,0 +1,104 @@
+name: Build and push DocsGPT FE Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    if: github.repository == 'arc53/DocsGPT'
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+            suffix: amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            suffix: arm64
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU # Only needed for emulation, not for native arm64 builds
+        if: matrix.platform == 'linux/arm64'
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push platform-specific images
+        uses: docker/build-push-action@v6
+        with:
+          file: './frontend/Dockerfile'
+          platforms: ${{ matrix.platform }}
+          context: ./frontend
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop-${{ matrix.suffix }}
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop-${{ matrix.suffix }}
+          provenance: false
+          sbom: false
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+          cache-to: type=inline
+
+  manifest:
+    if: github.repository == 'arc53/DocsGPT'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          install: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest for DockerHub
+        run: |
+          docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop-amd64 \
+            --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop-arm64
+          docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+
+      - name: Create and push manifest for ghcr.io
+        run: |
+          docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop-amd64 \
+            --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop-arm64
+          docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop

.github/workflows/labeler.yml

@@ -0,0 +1,16 @@
+# https://github.com/actions/labeler
+name: Pull Request Labeler
+on:
+  - pull_request_target
+jobs:
+  triage:
+    if: github.repository == 'arc53/DocsGPT'
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/labeler@v5
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          sync-labels: true

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Lint with Ruff
         uses: chartboost/ruff-action@v1

.github/workflows/pytest.yml

@@ -0,0 +1,30 @@
+name: Run python tests with pytest
+on: [push, pull_request]
+jobs:
+  pytest_and_coverage:
+    name: Run tests and count coverage
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd application
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          cd ../tests
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Test with pytest and generate coverage report
+        run: |
+          python -m pytest --cov=application --cov-report=xml --cov-report=term-missing
+      - name: Upload coverage reports to Codecov
+        if: github.event_name == 'pull_request' && matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/sync_fork.yaml

@@ -5,7 +5,7 @@ permissions:
 
 on:
   schedule:
-    - cron: "0 * * * *" # every hour
+    - cron: "0 0 * * *" # every hour
   workflow_dispatch:
 
 jobs:
@@ -17,7 +17,7 @@ jobs:
     steps:
       # Step 1: run a standard checkout action
       - name: Checkout target repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Step 2: run the sync action
       - name: Sync upstream changes

.github/workflows/vale.yml

@@ -0,0 +1,26 @@
+name: Vale Documentation Linter
+
+on:
+  pull_request:
+    paths:
+      - 'docs/**/*.md'
+      - 'docs/**/*.mdx'
+      - '**/*.md'
+      - '.vale.ini'
+      - '.github/styles/**'
+
+jobs:
+  vale:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Vale linter
+        uses: errata-ai/vale-action@v2
+        with:
+          files: docs
+          fail_on_error: false
+          version: 3.0.5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -2,10 +2,12 @@
 __pycache__/
 *.py[cod]
 *$py.class
+experiments/
 
+experiments
 # C extensions
 *.so
-
+*.next
 # Distribution / packaging
 .Python
 build/
@@ -75,6 +77,7 @@ target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
+**/*.ipynb
 
 # IPython
 profile_default/
@@ -112,6 +115,7 @@ venv.bak/
 # Spyder project settings
 .spyderproject
 .spyproject
+.jwt_secret_key
 
 # Rope project settings
 .ropeproject
@@ -169,4 +173,7 @@ application/vectors/
 
 **/yarn.lock
 
-node_modules/
+node_modules/
+.vscode/settings.json
+/models/
+model/

.vale.ini

@@ -0,0 +1,5 @@
+MinAlertLevel = warning
+StylesPath = .github/styles
+
+[*.{md,mdx}]
+BasedOnStyles = DocsGPT

.vscode/launch.json

@@ -0,0 +1,71 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Frontend Debug (npm)",
+      "type": "node-terminal",
+      "request": "launch",
+      "command": "npm run dev",
+      "cwd": "${workspaceFolder}/frontend"
+    },
+    {
+        "name": "Flask Debugger",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "flask",
+        "env": {
+            "FLASK_APP": "application/app.py",
+            "PYTHONPATH": "${workspaceFolder}",
+            "FLASK_ENV": "development",
+            "FLASK_DEBUG": "1",
+            "FLASK_RUN_PORT": "7091",
+            "FLASK_RUN_HOST": "0.0.0.0"
+
+        },
+        "args": [
+            "run",
+            "--no-debugger"
+        ],
+        "cwd": "${workspaceFolder}",
+    },
+    {
+      "name": "Celery Debugger",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}",
+      },
+      "args": [
+        "-A",
+        "application.app.celery",
+        "worker",
+        "-l",
+        "INFO",
+        "--pool=solo"
+      ],
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "name": "Dev Containers (Mongo + Redis)",
+      "type": "node-terminal",
+      "request": "launch",
+      "command": "docker compose -f deployment/docker-compose-dev.yaml up --build",
+      "cwd": "${workspaceFolder}"
+    }
+  ],
+  "compounds": [
+    {
+      "name": "DocsGPT: Full Stack",
+      "configurations": [
+        "Frontend Debug (npm)",
+        "Flask Debugger",
+        "Celery Debugger"
+      ],
+      "presentation": {
+        "group": "DocsGPT",
+        "order": 1
+      }
+    }
+  ]
+}

CODE_OF_CONDUCT.md

@@ -2,58 +2,58 @@
 
 ## Our Pledge
 
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
+We as members, contributors and leaders pledge to make participation in our
+community, a harassment-free experience for everyone, regardless of age, body
 size, visible or invisible disability, ethnicity, sex characteristics, gender
 identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, religion, or sexual identity
+nationality, personal appearance, race, religion or sexual identity
 and orientation.
 
 We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
+diverse, inclusive and a healthy community.
 
 ## Our Standards
 
-Examples of behavior that contributes to a positive environment for our
+Examples of behavior that contribute to a positive environment for our
 community include:
 
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the
-  overall community
+## Demonstrating empathy and kindness towards other people
+1. Being respectful and open to differing opinions, viewpoints, and experiences
+2. Giving and gracefully accepting constructive feedback
+3. Taking accountability and offering apologies to those who have been impacted by our errors,
+  while also gaining insights from the situation
+4. Focusing on what is best not just for us as individuals but for the
+  community as a whole
 
 Examples of unacceptable behavior include:
 
-* The use of sexualized language or imagery, and sexual attention or
+1. The use of sexualized language or imagery, and sexual attention or
   advances of any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email
+2. Trolling, insulting or derogatory comments, and personal or political attacks
+3. Public or private harassment
+4. Publishing other's private information, such as a physical or email
   address, without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
+5. Other conduct which could reasonably be considered inappropriate in a
   professional setting
 
 ## Enforcement Responsibilities
 
 Community leaders are responsible for clarifying and enforcing our standards of
 acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
+response to any behavior that they deem inappropriate, threatening, offensive
 or harmful.
 
 Community leaders have the right and responsibility to remove, edit, or reject
 comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
+not aligned to this Code of Conduct and will communicate reasons for moderation
 decisions when appropriate.
 
 ## Scope
 
-This Code of Conduct applies within all community spaces, and also applies when
+This Code of Conduct applies within all community spaces and also applies when
 an individual is officially representing the community in public spaces.
 Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
+posting via an official social media account or acting as an appointed
 representative at an online or offline event.
 
 ## Enforcement
@@ -63,29 +63,27 @@ reported to the community leaders responsible for enforcement at
 contact@arc53.com.
 All complaints will be reviewed and investigated promptly and fairly.
 
-All community leaders are obligated to respect the privacy and security of the
+All community leaders are obligated to be respectful towards the privacy and security of the
 reporter of any incident.
 
 ## Enforcement Guidelines
 
 Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
+the consequences for any action that they deem in violation of this Code of Conduct:
 
 ### 1. Correction
+* **Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community space.
 
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
+* **Consequence**: A private, written warning from community leaders, providing
 clarity around the nature of the violation and an explanation of why the
 behavior was inappropriate. A public apology may be requested.
 
 ### 2. Warning
-
-**Community Impact**: A violation through a single incident or series
+* **Community Impact**: A violation through a single incident or series
 of actions.
 
-**Consequence**: A warning with consequences for continued behavior. No
+* **Consequence**: A warning with consequences for continued behavior. No
 interaction with the people involved, including unsolicited interaction with
 those enforcing the Code of Conduct, for a specified period of time. This
 includes avoiding interactions in community spaces as well as external channels
@@ -93,23 +91,21 @@ like social media. Violating these terms may lead to a temporary or
 permanent ban.
 
 ### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
+* **Community Impact**: A serious violation of community standards, including
 sustained inappropriate behavior.
 
-**Consequence**: A temporary ban from any sort of interaction or public
+* **Consequence**: A temporary ban from any sort of interaction or public
 communication with the community for a specified period of time. No public or
 private interaction with the people involved, including unsolicited interaction
 with those enforcing the Code of Conduct, is allowed during this period.
 Violating these terms may lead to a permanent ban.
 
 ### 4. Permanent Ban
+* **Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,harassment of an
+individual or aggression towards or disparagement of classes of individuals.
 
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior,  harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within
+* **Consequence**: A permanent ban from any sort of public interaction within
 the community.
 
 ## Attribution

CONTRIBUTING.md

@@ -1,38 +1,151 @@
-# Welcome to DocsGPT Contributing guideline
+# Welcome to DocsGPT Contributing Guidelines
 
-Thank you for choosing this project to contribute to, we are all very grateful!
+Thank you for choosing to contribute to DocsGPT! We are all very grateful! 
 
 # We accept different types of contributions
 
-📣 Discussions - where you can start a new topic or answer some questions
+📣 **Discussions** - Engage in conversations, start new topics, or help answer questions.
 
-🐞 Issues - Is how we track tasks, sometimes its bugs that need fixing, sometimes its new features
+🐞 **Issues** - This is where we keep track of tasks. It could be bugs, fixes or suggestions for new features.
 
-🛠️ Pull requests - Is how you can suggest changes to our repository, to work on existing issue or to add new features
+🛠️ **Pull requests** - Suggest changes to our repository, either by working on existing issues or adding new features.
 
-📚 Wiki - where we have our documentation
+📚 **Wiki** - This is where our documentation resides.
 
 
 ## 🐞 Issues and Pull requests
 
-We value contributions to our issues in form of discussion or suggestion, we recommend that you check out existing issues and our [Roadmap](https://github.com/orgs/arc53/projects/2)
-
-If you want to contribute by writing code there are few things that you should know before doing it:
-We have frontend (React, Vite) and Backend (python)
-
-### If you are looking to contribute to Frontend (⚛️React, Vite):
-Current frontend is being migrated from /application to /frontend with a new design, so please contribute to the new on. Check out this [Milestone](https://github.com/arc53/DocsGPT/milestone/1) and its issues also [Figma](https://www.figma.com/file/OXLtrl1EAy885to6S69554/DocsGPT?node-id=0%3A1&t=hjWVuxRg9yi5YkJ9-1)
-Please try to follow guidelines
+- We value contributions in the form of discussions or suggestions. We recommend taking a look at existing issues and our [roadmap](https://github.com/orgs/arc53/projects/2).
 
 
-### If you are looking to contribute to Backend (🐍Python):
-Check out our issues, and contribute to /application or /scripts (ignore old  ingest_rst.py ingest_rst_sphinx.py files, they will be deprecated soon)
-Currently we don't have any tests(which would be useful😉) but before submitting you PR make sure that after you ingested some test data its queryable
+- If you're interested in contributing code, here are some important things to know:
 
-### Workflow:
-Create a fork, make changes on your forked repository, submit changes in a form of pull request
+- We have a frontend built on React (Vite) and a backend in Python.
 
-## Questions / collaboration
-Please join our [Discord](https://discord.gg/n5BX8dh8rU) don't hesitate, we are very friendly and welcoming to new contributors.
+  
+Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
 
-# Thank you so much for considering to contribute to DocsGPT!🙏
+### 👨‍💻 If you're interested in contributing code, here are some important things to know:
+
+For instructions on setting up a development environment, please refer to our [Development Deployment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).
+
+Tech Stack Overview:
+
+- 🌐 Frontend: Built with React (Vite) ⚛️,
+
+- 🖥 Backend: Developed in Python 🐍
+
+### 🌐 Frontend Contributions (⚛️ React, Vite)
+
+*   The updated Figma design can be found [here](https://www.figma.com/file/OXLtrl1EAy885to6S69554/DocsGPT?node-id=0%3A1&t=hjWVuxRg9yi5YkJ9-1).  Please try to follow the guidelines.
+*   **Coding Style:** We follow a strict coding style enforced by ESLint and Prettier. Please ensure your code adheres to the configuration provided in our repository's `fronetend/.eslintrc.js` file.  We recommend configuring your editor with ESLint and Prettier to help with this.
+* **Component Structure:** Strive for small, reusable components.  Favor functional components and hooks over class components where possible.
+* **State Management** If you need to add stores, please use Redux.
+
+### 🖥 Backend Contributions (🐍 Python)
+
+- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) 
+- All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
+- Before submitting your Pull Request, ensure it can be queried after ingesting some test data.
+- **Coding Style:** We adhere to the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style guide for Python code. We use `ruff` as our linter and code formatter.  Please ensure your code is formatted correctly and passes `ruff` checks before submitting.
+- **Type Hinting:**  Please use type hints for all function arguments and return values. This improves code readability and helps catch errors early.  Example:
+
+    ```python
+    def my_function(name: str, count: int) -> list[str]:
+        ...
+    ```
+- **Docstrings:**  All functions and classes should have docstrings explaining their purpose, parameters, and return values.  We prefer the [Google style docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). Example:
+
+    ```python
+    def my_function(name: str, count: int) -> list[str]:
+        """Does something with a name and a count.
+
+        Args:
+            name: The name to use.
+            count: The number of times to do it.
+
+        Returns:
+            A list of strings.
+        """
+        ...
+    ```
+  
+### Testing
+
+To run unit tests from the root of the repository, execute:
+```
+python -m pytest
+```
+
+## Workflow 📈
+
+Here's a step-by-step guide on how to contribute to DocsGPT:
+
+1. **Fork the Repository:**
+   - Click the "Fork" button at the top-right of this repository to create your fork.
+
+2. **Clone the Forked Repository:**
+   - Clone the repository using:
+      ``` shell
+      git clone https://github.com/<your-github-username>/DocsGPT.git
+      ```
+
+3. **Keep your Fork in Sync:**
+   - Before you make any changes, make sure that your fork is in sync to avoid merge conflicts using:
+     ```shell
+     git remote add upstream https://github.com/arc53/DocsGPT.git
+     git pull upstream main
+     ```
+
+4. **Create and Switch to a New Branch:**
+   - Create a new branch for your contribution using:
+     ```shell
+     git checkout -b your-branch-name
+     ```
+
+5. **Make Changes:**
+   - Make the required changes in your branch.
+
+6. **Add Changes to the Staging Area:**
+   - Add your changes to the staging area using:
+     ```shell
+     git add .
+     ```
+
+7. **Commit Your Changes:**
+   - Commit your changes with a descriptive commit message using:
+     ```shell
+     git commit -m "Your descriptive commit message"
+     ```
+
+8. **Push Your Changes to the Remote Repository:**
+   - Push your branch with changes to your fork on GitHub using:
+     ```shell
+     git push origin your-branch-name
+     ```
+
+9. **Submit a Pull Request (PR):**
+   - Create a Pull Request from your branch to the main repository. Make sure to include a detailed description of your changes and reference any related issues.
+
+10. **Collaborate:**
+   - Be responsive to comments and feedback on your PR.
+   - Make necessary updates as suggested.
+   - Once your PR is approved, it will be merged into the main repository.
+
+11. **Testing:**
+   - Before submitting a Pull Request, ensure your code passes all unit tests.
+   - To run unit tests from the root of the repository, execute:
+     ```shell
+     python -m pytest
+     ```
+
+*Note: You should run the unit test only after making the changes to the backend code.*
+
+12. **Questions and Collaboration:**
+    - Feel free to join our Discord. We're very friendly and welcoming to new contributors, so don't hesitate to reach out.
+
+Thank you for considering contributing to DocsGPT! 🙏
+
+## Questions/collaboration
+Feel free to join our [Discord](https://discord.gg/vN7YFfdMpj). We're very friendly and welcoming to new contributors, so don't hesitate to reach out.
+# Thank you so much for considering to contributing DocsGPT!🙏

HACKTOBERFEST.md

@@ -0,0 +1,39 @@
+# **🎉 Join the Hacktoberfest with DocsGPT and win a Free T-shirt for a meaningful PR! 🎉**
+
+Welcome, contributors! We're excited to announce that DocsGPT is participating in Hacktoberfest. Get involved by submitting meaningful pull requests.
+
+All Meaningful contributors with accepted PRs that were created for issues with the `hacktoberfest` label (set by our maintainer team: dartpain, siiddhantt, pabik, ManishMadan2882) will receive a cool T-shirt! 🤩.
+<img width="1331" height="678" alt="hacktoberfest-mocks-preview" src="https://github.com/user-attachments/assets/633f6377-38db-48f5-b519-a8b3855a9eb4" />
+
+Fill in [this form](https://forms.gle/Npaba4n9Epfyx56S8
+) after your PR was merged please 
+
+If you are in doubt don't hesitate to ping us on discord, ping me - Alex (dartpain).
+
+## 📜 Here's How to Contribute:
+```text
+🛠️ Code: This is the golden ticket! Make meaningful contributions through PRs.
+
+🧩 API extension: Build an app utilising DocsGPT API. We prefer submissions that showcase original ideas and turn the API into an AI agent.
+They can be a completely separate repos. 
+For example: 
+https://github.com/arc53/tg-bot-docsgpt-extenstion or 
+https://github.com/arc53/DocsGPT-cli
+
+Non-Code Contributions:
+
+📚 Wiki: Improve our documentation, create a guide.
+
+🖥️ Design: Improve the UI/UX or design a new feature.
+```
+
+### 📝 Guidelines for Pull Requests:
+- Familiarize yourself with the current contributions and our [Roadmap](https://github.com/orgs/arc53/projects/2).
+- Before contributing check existing [issues](https://github.com/arc53/DocsGPT/issues) or [create](https://github.com/arc53/DocsGPT/issues/new/choose) an issue and wait to get assigned.
+- Once you are finished with your contribution, please fill in this [form](https://forms.gle/Npaba4n9Epfyx56S8).
+- Refer to the [Documentation](https://docs.docsgpt.cloud/).
+- Feel free to join our [Discord](https://discord.gg/vN7YFfdMpj) server. We're here to help newcomers, so don't hesitate to jump in! Join us [here](https://discord.gg/vN7YFfdMpj).
+  
+Thank you very much for considering contributing to DocsGPT during Hacktoberfest! 🙏 Your contributions (not just simple typos) could earn you a stylish new t-shirt.
+
+We will publish a t-shirt design later into the October.

README.md

@@ -3,102 +3,164 @@
 </h1>
 
 <p align="center">
-  <strong>Open-Source Documentation Assistant</strong>
+  <strong>Private AI for agents, assistants and enterprise search</strong>
 </p>
 
 <p align="left">
-  <strong>DocsGPT</strong> is a cutting-edge open-source solution that streamlines the process of finding information in project documentation. With its integration of the powerful <strong>GPT</strong> models, developers can easily ask questions about a project and receive accurate answers.
-  
-Say goodbye to time-consuming manual searches, and let <strong>DocsGPT</strong> help you quickly find the information you need. Try it out and see how it revolutionizes your project documentation experience. Contribute to its development and be a part of the future of AI-powered assistance.
+  <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> is an open-source AI platform for building intelligent agents and assistants. Features Agent Builder, deep research tools, document analysis (PDF, Office, web content), Multi-model support (choose your provider or run locally), and rich API connectivity for agents with actionable tools and integrations. Deploy anywhere with complete privacy control.
 </p>
 
 <div align="center">
   
-  <a href="https://discord.gg/n5BX8dh8rU">![example1](https://img.shields.io/github/stars/arc53/docsgpt?style=social)</a>
-  <a href="https://discord.gg/n5BX8dh8rU">![example2](https://img.shields.io/github/forks/arc53/docsgpt?style=social)</a>
-  <a href="https://discord.gg/n5BX8dh8rU">![example3](https://img.shields.io/github/license/arc53/docsgpt)</a>
-  <a href="https://discord.gg/n5BX8dh8rU">![example3](https://img.shields.io/discord/1070046503302877216)</a>
-  
+  <a href="https://github.com/arc53/DocsGPT">![link to main GitHub showing Stars number](https://img.shields.io/github/stars/arc53/docsgpt?style=social)</a>
+  <a href="https://github.com/arc53/DocsGPT">![link to main GitHub showing Forks number](https://img.shields.io/github/forks/arc53/docsgpt?style=social)</a>
+  <a href="https://github.com/arc53/DocsGPT/blob/main/LICENSE">![link to license file](https://img.shields.io/github/license/arc53/docsgpt)</a>
+  <a href="https://www.bestpractices.dev/projects/9907"><img src="https://www.bestpractices.dev/projects/9907/badge"></a>
+  <a href="https://discord.gg/vN7YFfdMpj">![link to discord](https://img.shields.io/discord/1070046503302877216)</a>
+  <a href="https://x.com/docsgptai">![X (formerly Twitter) URL](https://img.shields.io/twitter/follow/docsgptai)</a>
+
+<a href="https://docs.docsgpt.cloud/quickstart">⚡️ Quickstart</a> • <a href="https://app.docsgpt.cloud/">☁️ Cloud Version</a> • <a href="https://discord.gg/vN7YFfdMpj">💬 Discord</a>
+<br>
+<a href="https://docs.docsgpt.cloud/">📖 Documentation</a> • <a href="https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md">👫 Contribute</a> • <a href="https://blog.docsgpt.cloud/">🗞 Blog</a>
+<br>
+
 </div>
 
-![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
-
-
-## Features
-
-![Group 9](https://user-images.githubusercontent.com/17906039/220427472-2644cff4-7666-46a5-819f-fc4a521f63c7.png)
-
 
+<div align="center">
+  <br>
+<img src="https://d3dg1063dc54p9.cloudfront.net/videos/demov7.gif" alt="video-example-of-docs-gpt" width="800" height="450">
+</div>
+<h3 align="left">
+  <strong>Key Features:</strong>
+</h3>
+<ul align="left">
+    <li><strong>🗂️ Wide Format Support:</strong> Reads PDF, DOCX, CSV, XLSX, EPUB, MD, RST, HTML, MDX, JSON, PPTX, and images.</li>
+    <li><strong>🌐 Web & Data Integration:</strong> Ingests from URLs, sitemaps, Reddit, GitHub and web crawlers.</li>
+    <li><strong>✅ Reliable Answers:</strong> Get accurate, hallucination-free responses with source citations viewable in a clean UI.</li>
+    <li><strong>🔑 Streamlined API Keys:</strong>  Generate keys linked to your settings, documents, and models, simplifying chatbot and integration setup.</li>
+    <li><strong>🔗 Actionable Tooling:</strong> Connect to APIs, tools, and other services to enable LLM actions.</li>
+    <li><strong>🧩 Pre-built Integrations:</strong> Use readily available HTML/React chat widgets, search tools, Discord/Telegram bots, and more.</li>
+    <li><strong>🔌 Flexible Deployment:</strong> Works with major LLMs (OpenAI, Google, Anthropic) and local models (Ollama, llama_cpp).</li>
+    <li><strong>🏢 Secure & Scalable:</strong> Run privately and securely with Kubernetes support, designed for enterprise-grade reliability.</li>
+</ul>
 
 ## Roadmap
 
-You can find our [Roadmap](https://github.com/orgs/arc53/projects/2) here, please don't hesitate contributing or creating issues, it helps us make DocsGPT better!
+- [x] Full GoogleAI compatibility (Jan 2025)
+- [x] Add tools (Jan 2025)
+- [x] Manually updating chunks in the app UI (Feb 2025)
+- [x] Devcontainer for easy development (Feb 2025)
+- [x] ReACT agent (March 2025)
+- [x] Chatbots menu re-design to handle tools, agent types, and more (April 2025)
+- [x] New input box in the conversation menu (April 2025)
+- [x] Add triggerable actions / tools (webhook) (April 2025)
+- [x] Agent optimisations (May 2025)
+- [x] Filesystem sources update (July 2025)
+- [x] Json Responses (August 2025)
+- [x] MCP support (August 2025)
+- [x] Google Drive integration (September 2025)
+- [x] Add OAuth 2.0 authentication for MCP (September 2025)
+- [ ] SharePoint integration (October 2025)
+- [ ] Deep Agents (October 2025)
+- [ ] Agent scheduling
 
+You can find our full roadmap [here](https://github.com/orgs/arc53/projects/2). Please don't hesitate to contribute or create issues, it helps us improve DocsGPT!
 
+### Production Support / Help for Companies:
 
-## [Live preview](https://docsgpt.arc53.com/)
+We're eager to provide personalized assistance when deploying your DocsGPT to a live environment.
 
-## [Join Our Discord](https://discord.gg/n5BX8dh8rU)
+[Get a Demo :wave:](https://www.docsgpt.cloud/contact)⁠
 
+[Send Email :email:](mailto:support@docsgpt.cloud?subject=DocsGPT%20support%2Fsolutions)
 
-## Project structure
-- Application - flask app (main application)
+## Join the Lighthouse Program 🌟
 
-- Extensions - chrome extension
+Calling all developers and GenAI innovators! The **DocsGPT Lighthouse Program** connects technical leaders actively deploying or extending DocsGPT in real-world scenarios. Collaborate directly with our team to shape the roadmap, access priority support, and build enterprise-ready solutions with exclusive community insights.
 
-- Scripts - script that creates similarity search index and store for other libraries. 
-
-- frontend - frontend in vite and
+[Learn More & Apply →](https://docs.google.com/forms/d/1KAADiJinUJ8EMQyfTXUIGyFbqINNClNR3jBNWq7DgTE)
 
 ## QuickStart
 
-Note: Make sure you have docker installed
+> [!Note]
+> Make sure you have [Docker](https://docs.docker.com/engine/install/) installed
 
-1. Open dowload this repository with `git clone https://github.com/arc53/DocsGPT.git`
-2. Create .env file in your root directory and set your OPENAI_API_KEY with your openai api key and  VITE_API_STREAMING to true or false if you dont want streaming answers
-3. Run `docker-compose build && docker-compose up`
-4. Navigate to http://localhost:5173/
+A more detailed [Quickstart](https://docs.docsgpt.cloud/quickstart) is available in our documentation
 
-To stop just run Ctrl + C
+1. **Clone the repository:**
 
-## Development environments
+   ```bash
+   git clone https://github.com/arc53/DocsGPT.git
+   cd DocsGPT
+   ```
 
-Spin up only 2 containers from docker-compose.yaml (by deleting all services except for redis and mongo)
+**For macOS and Linux:**
 
-Make sure you have python 3.10 or 3.11 installed
+2. **Run the setup script:**
 
-1. Navigate to `/application` folder
-2. Run `docker-compose -f docker-compose-dev.yaml build && docker-compose -f docker-compose-dev.yaml up -d`
-3. Export required variables              
-`export CELERY_BROKER_URL=redis://localhost:6379/0`   
-`export CELERY_RESULT_BACKEND=redis://localhost:6379/1`
-`export MONGO_URI=mongodb://localhost:27017/docsgpt`
-4. Install dependencies
-`pip install -r requirements.txt`
-5. Prepare .env file
-Copy .env_sample and create .env with your openai api token
-6. Run the app
-`python wsgi.py`
-7. Start worker with `celery -A app.celery worker -l INFO`
+   ```bash
+   ./setup.sh
+   ```
 
-To start frontend
-1. Navigate to `/frontend` folder
-2. Install dependencies
-`npm install`
-3. Run the app
-4. `npm run dev`
+**For Windows:**
 
+2. **Run the PowerShell setup script:**
 
-[How to install the Chrome extension](https://github.com/arc53/docsgpt/wiki#launch-chrome-extension)
+   ```powershell
+   PowerShell -ExecutionPolicy Bypass -File .\setup.ps1
+   ```
 
+Either script will guide you through setting up DocsGPT. Five options available: using the public API, running locally, connecting to a local inference engine, using a cloud API provider, or build the docker image locally. Scripts will automatically configure your `.env` file and handle necessary downloads and installations based on your chosen option.
 
-## [Guides](https://github.com/arc53/docsgpt/wiki)
+**Navigate to http://localhost:5173/**
 
-## [Interested in contributing?](https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md)
+To stop DocsGPT, open a terminal in the `DocsGPT` directory and run:
 
-## [How to use any other documentation](https://github.com/arc53/docsgpt/wiki/How-to-train-on-other-documentation)
+```bash
+docker compose -f deployment/docker-compose.yaml down
+```
 
-## [How to host it locally (so all data will stay on-premises)](https://github.com/arc53/DocsGPT/wiki/How-to-use-different-LLM's#hosting-everything-locally)
+(or use the specific `docker compose down` command shown after running the setup script).
 
-Built with [🦜️🔗 LangChain](https://github.com/hwchase17/langchain)
+> [!Note]
+> For development environment setup instructions, please refer to the [Development Environment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).
 
+## Contributing
+
+Please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file for information about how to get involved. We welcome issues, questions, and pull requests.
+
+## Architecture
+
+![Architecture chart](https://github.com/user-attachments/assets/fc6a7841-ddfc-45e6-b5a0-d05fe648cbe2)
+
+## Project Structure
+
+- Application - Flask app (main application).
+
+- Extensions - Extensions, like react widget or discord bot.
+
+- Frontend - Frontend uses <a href="https://vitejs.dev/">Vite</a> and <a href="https://react.dev/">React</a>.
+
+- Scripts - Miscellaneous scripts.
+
+## Code Of Conduct
+
+We as members, contributors, and leaders, pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. Please refer to the [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) file for more information about contributing.
+
+## Many Thanks To Our Contributors⚡
+
+<a href="https://github.com/arc53/DocsGPT/graphs/contributors" alt="View Contributors">
+  <img src="https://contrib.rocks/image?repo=arc53/DocsGPT" alt="Contributors" />
+</a>
+
+## License
+
+The source code license is [MIT](https://opensource.org/license/mit/), as described in the [LICENSE](LICENSE) file.
+
+<p>This project is supported by:</p>
+<p>
+  <a href="https://www.digitalocean.com/?utm_medium=opensource&utm_source=DocsGPT">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px">
+  </a>
+</p>

SECURITY.md

@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+Supported Versions:
+
+Currently, we support security patches by committing changes and bumping the version published on Github.
+
+## Reporting a Vulnerability
+
+Found a vulnerability? Please email us:
+
+security@arc53.com
+

application/.env_sample

@@ -1,6 +1,11 @@
 API_KEY=your_api_key
 EMBEDDINGS_KEY=your_api_key
-CELERY_BROKER_URL=redis://localhost:6379/0
-CELERY_RESULT_BACKEND=redis://localhost:6379/1
-MONGO_URI=mongodb://localhost:27017/docsgpt
-API_URL=http://localhost:5001
+API_URL=http://localhost:7091
+FLASK_APP=application/app.py
+FLASK_DEBUG=true
+
+#For OPENAI on Azure
+OPENAI_API_BASE=
+OPENAI_API_VERSION=
+AZURE_DEPLOYMENT_NAME=
+AZURE_EMBEDDINGS_DEPLOYMENT_NAME=

application/Dockerfile

@@ -1,25 +1,87 @@
-FROM python:3.10-slim-bullseye as builder
+# Builder Stage
+FROM ubuntu:24.04 as builder
 
-# Tiktoken requires Rust toolchain, so build it in a separate stage
-RUN apt-get update && apt-get install -y gcc curl
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && apt-get install --reinstall libc6-dev -y
-ENV PATH="/root/.cargo/bin:${PATH}"
-RUN pip install --upgrade pip && pip install tiktoken==0.3.3
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends gcc wget unzip libc6-dev python3.12 python3.12-venv && \
+    rm -rf /var/lib/apt/lists/* 
+
+# Verify Python installation and setup symlink
+RUN if [ -f /usr/bin/python3.12 ]; then \
+        ln -s /usr/bin/python3.12 /usr/bin/python; \
+    else \
+        echo "Python 3.12 not found"; exit 1; \
+    fi
+
+# Download and unzip the model
+RUN wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip && \
+    unzip mpnet-base-v2.zip -d models && \
+    rm mpnet-base-v2.zip
+
+# Install Rust
+RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y
+
+# Clean up to reduce container size
+RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements.txt
 COPY requirements.txt .
-RUN pip install -r requirements.txt
 
+# Setup Python virtual environment
+RUN python3.12 -m venv /venv
 
-FROM python:3.10-slim-bullseye
-# Copy pre-built packages from builder stage
-COPY --from=builder /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/
-RUN pip install gunicorn==20.1.0
-RUN pip install celery==5.2.7
+# Activate virtual environment and install Python packages
+ENV PATH="/venv/bin:$PATH"
+
+# Install Python packages
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir tiktoken && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Final Stage
+FROM ubuntu:24.04 as final
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && apt-get install -y --no-install-recommends python3.12 && \
+    ln -s /usr/bin/python3.12 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set working directory
 WORKDIR /app
-COPY . /app
-ENV FLASK_APP=app.py
-ENV FLASK_DEBUG=true
 
+# Create a non-root user: `appuser` (Feel free to choose a name)
+RUN groupadd -r appuser && \
+    useradd -r -g appuser -d /app -s /sbin/nologin -c "Docker image user" appuser
 
-EXPOSE 5001
+# Copy the virtual environment and model from the builder stage
+COPY --from=builder /venv /venv
 
-CMD ["gunicorn", "-w", "2", "--timeout", "120", "--bind", "0.0.0.0:5001", "wsgi:app"]
+COPY --from=builder /models /app/models
+
+# Copy your application code
+COPY . /app/application
+
+# Change the ownership of the /app directory to the appuser
+	
+RUN mkdir -p /app/application/inputs/local
+RUN chown -R appuser:appuser /app
+
+# Set environment variables
+ENV FLASK_APP=app.py \
+    FLASK_DEBUG=true \
+    PATH="/venv/bin:$PATH"
+
+# Expose the port the app runs on
+EXPOSE 7091
+
+# Switch to non-root user
+USER appuser
+
+# Start Gunicorn
+CMD ["gunicorn", "-w", "1", "--timeout", "120", "--bind", "0.0.0.0:7091", "--preload", "application.wsgi:app"]

application/agents/agent_creator.py

@@ -0,0 +1,20 @@
+from application.agents.classic_agent import ClassicAgent
+from application.agents.react_agent import ReActAgent
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class AgentCreator:
+    agents = {
+        "classic": ClassicAgent,
+        "react": ReActAgent,
+    }
+
+    @classmethod
+    def create_agent(cls, type, *args, **kwargs):
+        agent_class = cls.agents.get(type.lower())
+        if not agent_class:
+            raise ValueError(f"No agent class found for type {type}")
+        
+        return agent_class(*args, **kwargs)

application/agents/base.py

@@ -0,0 +1,472 @@
+import logging
+import uuid
+from abc import ABC, abstractmethod
+from typing import Dict, Generator, List, Optional
+
+from bson.objectid import ObjectId
+
+from application.agents.tools.tool_action_parser import ToolActionParser
+from application.agents.tools.tool_manager import ToolManager
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.llm.handlers.handler_creator import LLMHandlerCreator
+from application.llm.llm_creator import LLMCreator
+from application.logging import build_stack_data, log_activity, LogContext
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAgent(ABC):
+    def __init__(
+        self,
+        endpoint: str,
+        llm_name: str,
+        model_id: str,
+        api_key: str,
+        user_api_key: Optional[str] = None,
+        prompt: str = "",
+        chat_history: Optional[List[Dict]] = None,
+        retrieved_docs: Optional[List[Dict]] = None,
+        decoded_token: Optional[Dict] = None,
+        attachments: Optional[List[Dict]] = None,
+        json_schema: Optional[Dict] = None,
+        limited_token_mode: Optional[bool] = False,
+        token_limit: Optional[int] = settings.DEFAULT_AGENT_LIMITS["token_limit"],
+        limited_request_mode: Optional[bool] = False,
+        request_limit: Optional[int] = settings.DEFAULT_AGENT_LIMITS["request_limit"],
+        compressed_summary: Optional[str] = None,
+    ):
+        self.endpoint = endpoint
+        self.llm_name = llm_name
+        self.model_id = model_id
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+        self.prompt = prompt
+        self.decoded_token = decoded_token or {}
+        self.user: str = self.decoded_token.get("sub")
+        self.tool_config: Dict = {}
+        self.tools: List[Dict] = []
+        self.tool_calls: List[Dict] = []
+        self.chat_history: List[Dict] = chat_history if chat_history is not None else []
+        self.llm = LLMCreator.create_llm(
+            llm_name,
+            api_key=api_key,
+            user_api_key=user_api_key,
+            decoded_token=decoded_token,
+            model_id=model_id,
+        )
+        self.retrieved_docs = retrieved_docs or []
+        self.llm_handler = LLMHandlerCreator.create_handler(
+            llm_name if llm_name else "default"
+        )
+        self.attachments = attachments or []
+        self.json_schema = json_schema
+        self.limited_token_mode = limited_token_mode
+        self.token_limit = token_limit
+        self.limited_request_mode = limited_request_mode
+        self.request_limit = request_limit
+        self.compressed_summary = compressed_summary
+        self.current_token_count = 0
+        self.context_limit_reached = False
+
+    @log_activity()
+    def gen(
+        self, query: str, log_context: LogContext = None
+    ) -> Generator[Dict, None, None]:
+        yield from self._gen_inner(query, log_context)
+
+    @abstractmethod
+    def _gen_inner(
+        self, query: str, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        pass
+
+    def _get_tools(self, api_key: str = None) -> Dict[str, Dict]:
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        agents_collection = db["agents"]
+        tools_collection = db["user_tools"]
+
+        agent_data = agents_collection.find_one({"key": api_key or self.user_api_key})
+        tool_ids = agent_data.get("tools", []) if agent_data else []
+
+        tools = (
+            tools_collection.find(
+                {"_id": {"$in": [ObjectId(tool_id) for tool_id in tool_ids]}}
+            )
+            if tool_ids
+            else []
+        )
+        tools = list(tools)
+        tools_by_id = {str(tool["_id"]): tool for tool in tools} if tools else {}
+
+        return tools_by_id
+
+    def _get_user_tools(self, user="local"):
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        user_tools_collection = db["user_tools"]
+        user_tools = user_tools_collection.find({"user": user, "status": True})
+        user_tools = list(user_tools)
+
+        return {str(i): tool for i, tool in enumerate(user_tools)}
+
+    def _build_tool_parameters(self, action):
+        params = {"type": "object", "properties": {}, "required": []}
+        for param_type in ["query_params", "headers", "body", "parameters"]:
+            if param_type in action and action[param_type].get("properties"):
+                for k, v in action[param_type]["properties"].items():
+                    if v.get("filled_by_llm", True):
+                        params["properties"][k] = {
+                            key: value
+                            for key, value in v.items()
+                            if key != "filled_by_llm" and key != "value"
+                        }
+
+                        params["required"].append(k)
+        return params
+
+    def _prepare_tools(self, tools_dict):
+        self.tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": f"{action['name']}_{tool_id}",
+                    "description": action["description"],
+                    "parameters": self._build_tool_parameters(action),
+                },
+            }
+            for tool_id, tool in tools_dict.items()
+            if (
+                (tool["name"] == "api_tool" and "actions" in tool.get("config", {}))
+                or (tool["name"] != "api_tool" and "actions" in tool)
+            )
+            for action in (
+                tool["config"]["actions"].values()
+                if tool["name"] == "api_tool"
+                else tool["actions"]
+            )
+            if action.get("active", True)
+        ]
+
+    def _execute_tool_action(self, tools_dict, call):
+        parser = ToolActionParser(self.llm.__class__.__name__)
+        tool_id, action_name, call_args = parser.parse_args(call)
+
+        call_id = getattr(call, "id", None) or str(uuid.uuid4())
+
+        # Check if parsing failed
+
+        if tool_id is None or action_name is None:
+            error_message = f"Error: Failed to parse LLM tool call. Tool name: {getattr(call, 'name', 'unknown')}"
+            logger.error(error_message)
+
+            tool_call_data = {
+                "tool_name": "unknown",
+                "call_id": call_id,
+                "action_name": getattr(call, "name", "unknown"),
+                "arguments": call_args or {},
+                "result": f"Failed to parse tool call. Invalid tool name format: {getattr(call, 'name', 'unknown')}",
+            }
+            yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
+            self.tool_calls.append(tool_call_data)
+            return "Failed to parse tool call.", call_id
+        # Check if tool_id exists in available tools
+
+        if tool_id not in tools_dict:
+            error_message = f"Error: Tool ID '{tool_id}' extracted from LLM call not found in available tools_dict. Available IDs: {list(tools_dict.keys())}"
+            logger.error(error_message)
+
+            # Return error result
+
+            tool_call_data = {
+                "tool_name": "unknown",
+                "call_id": call_id,
+                "action_name": f"{action_name}_{tool_id}",
+                "arguments": call_args,
+                "result": f"Tool with ID {tool_id} not found. Available tools: {list(tools_dict.keys())}",
+            }
+            yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
+            self.tool_calls.append(tool_call_data)
+            return f"Tool with ID {tool_id} not found.", call_id
+        tool_call_data = {
+            "tool_name": tools_dict[tool_id]["name"],
+            "call_id": call_id,
+            "action_name": f"{action_name}_{tool_id}",
+            "arguments": call_args,
+        }
+        yield {"type": "tool_call", "data": {**tool_call_data, "status": "pending"}}
+
+        tool_data = tools_dict[tool_id]
+        action_data = (
+            tool_data["config"]["actions"][action_name]
+            if tool_data["name"] == "api_tool"
+            else next(
+                action
+                for action in tool_data["actions"]
+                if action["name"] == action_name
+            )
+        )
+
+        query_params, headers, body, parameters = {}, {}, {}, {}
+        param_types = {
+            "query_params": query_params,
+            "headers": headers,
+            "body": body,
+            "parameters": parameters,
+        }
+
+        for param_type, target_dict in param_types.items():
+            if param_type in action_data and action_data[param_type].get("properties"):
+                for param, details in action_data[param_type]["properties"].items():
+                    if param not in call_args and "value" in details:
+                        target_dict[param] = details["value"]
+        for param, value in call_args.items():
+            for param_type, target_dict in param_types.items():
+                if param_type in action_data and param in action_data[param_type].get(
+                    "properties", {}
+                ):
+                    target_dict[param] = value
+        tm = ToolManager(config={})
+
+        # Prepare tool_config and add tool_id for memory tools
+
+        if tool_data["name"] == "api_tool":
+            tool_config = {
+                "url": tool_data["config"]["actions"][action_name]["url"],
+                "method": tool_data["config"]["actions"][action_name]["method"],
+                "headers": headers,
+                "query_params": query_params,
+            }
+        else:
+            tool_config = tool_data["config"].copy() if tool_data["config"] else {}
+            # Add tool_id from MongoDB _id for tools that need instance isolation (like memory tool)
+            # Use MongoDB _id if available, otherwise fall back to enumerated tool_id
+
+            tool_config["tool_id"] = str(tool_data.get("_id", tool_id))
+        tool = tm.load_tool(
+            tool_data["name"],
+            tool_config=tool_config,
+            user_id=self.user,  # Pass user ID for MCP tools credential decryption
+        )
+        if tool_data["name"] == "api_tool":
+            print(
+                f"Executing api: {action_name} with query_params: {query_params}, headers: {headers}, body: {body}"
+            )
+            result = tool.execute_action(action_name, **body)
+        else:
+            print(f"Executing tool: {action_name} with args: {call_args}")
+            result = tool.execute_action(action_name, **parameters)
+        tool_call_data["result"] = (
+            f"{str(result)[:50]}..." if len(str(result)) > 50 else result
+        )
+
+        yield {"type": "tool_call", "data": {**tool_call_data, "status": "completed"}}
+        self.tool_calls.append(tool_call_data)
+
+        return result, call_id
+
+    def _get_truncated_tool_calls(self):
+        return [
+            {
+                **tool_call,
+                "result": (
+                    f"{str(tool_call['result'])[:50]}..."
+                    if len(str(tool_call["result"])) > 50
+                    else tool_call["result"]
+                ),
+                "status": "completed",
+            }
+            for tool_call in self.tool_calls
+        ]
+
+    def _calculate_current_context_tokens(self, messages: List[Dict]) -> int:
+        """
+        Calculate total tokens in current context (messages).
+
+        Args:
+            messages: List of message dicts
+
+        Returns:
+            Total token count
+        """
+        from application.api.answer.services.compression.token_counter import (
+            TokenCounter,
+        )
+
+        return TokenCounter.count_message_tokens(messages)
+
+    def _check_context_limit(self, messages: List[Dict]) -> bool:
+        """
+        Check if we're approaching context limit (80%).
+
+        Args:
+            messages: Current message list
+
+        Returns:
+            True if at or above 80% of context limit
+        """
+        from application.core.model_utils import get_token_limit
+        from application.core.settings import settings
+
+        try:
+            # Calculate current tokens
+            current_tokens = self._calculate_current_context_tokens(messages)
+            self.current_token_count = current_tokens
+
+            # Get context limit for model
+            context_limit = get_token_limit(self.model_id)
+
+            # Calculate threshold (80%)
+            threshold = int(context_limit * settings.COMPRESSION_THRESHOLD_PERCENTAGE)
+
+            # Check if we've reached the limit
+            if current_tokens >= threshold:
+                logger.warning(
+                    f"Context limit approaching: {current_tokens}/{context_limit} tokens "
+                    f"({(current_tokens/context_limit)*100:.1f}%)"
+                )
+                return True
+
+            return False
+
+        except Exception as e:
+            logger.error(f"Error checking context limit: {str(e)}", exc_info=True)
+            return False
+
+    def _build_messages(
+        self,
+        system_prompt: str,
+        query: str,
+    ) -> List[Dict]:
+        """Build messages using pre-rendered system prompt"""
+        # Append compression summary to system prompt if present
+        if self.compressed_summary:
+            compression_context = (
+                "\n\n---\n\n"
+                "This session is being continued from a previous conversation that "
+                "has been compressed to fit within context limits. "
+                "The conversation is summarized below:\n\n"
+                f"{self.compressed_summary}"
+            )
+            system_prompt = system_prompt + compression_context
+
+        messages = [{"role": "system", "content": system_prompt}]
+
+        for i in self.chat_history:
+            if "prompt" in i and "response" in i:
+                messages.append({"role": "user", "content": i["prompt"]})
+                messages.append({"role": "assistant", "content": i["response"]})
+            if "tool_calls" in i:
+                for tool_call in i["tool_calls"]:
+                    call_id = tool_call.get("call_id") or str(uuid.uuid4())
+
+                    function_call_dict = {
+                        "function_call": {
+                            "name": tool_call.get("action_name"),
+                            "args": tool_call.get("arguments"),
+                            "call_id": call_id,
+                        }
+                    }
+                    function_response_dict = {
+                        "function_response": {
+                            "name": tool_call.get("action_name"),
+                            "response": {"result": tool_call.get("result")},
+                            "call_id": call_id,
+                        }
+                    }
+
+                    messages.append(
+                        {"role": "assistant", "content": [function_call_dict]}
+                    )
+                    messages.append(
+                        {"role": "tool", "content": [function_response_dict]}
+                    )
+        messages.append({"role": "user", "content": query})
+        return messages
+
+    def _llm_gen(self, messages: List[Dict], log_context: Optional[LogContext] = None):
+        gen_kwargs = {"model": self.model_id, "messages": messages}
+
+        if (
+            hasattr(self.llm, "_supports_tools")
+            and self.llm._supports_tools
+            and self.tools
+        ):
+            gen_kwargs["tools"] = self.tools
+        if (
+            self.json_schema
+            and hasattr(self.llm, "_supports_structured_output")
+            and self.llm._supports_structured_output()
+        ):
+            structured_format = self.llm.prepare_structured_output_format(
+                self.json_schema
+            )
+            if structured_format:
+                if self.llm_name == "openai":
+                    gen_kwargs["response_format"] = structured_format
+                elif self.llm_name == "google":
+                    gen_kwargs["response_schema"] = structured_format
+        resp = self.llm.gen_stream(**gen_kwargs)
+
+        if log_context:
+            data = build_stack_data(self.llm, exclude_attributes=["client"])
+            log_context.stacks.append({"component": "llm", "data": data})
+        return resp
+
+    def _llm_handler(
+        self,
+        resp,
+        tools_dict: Dict,
+        messages: List[Dict],
+        log_context: Optional[LogContext] = None,
+        attachments: Optional[List[Dict]] = None,
+    ):
+        resp = self.llm_handler.process_message_flow(
+            self, resp, tools_dict, messages, attachments, True
+        )
+        if log_context:
+            data = build_stack_data(self.llm_handler, exclude_attributes=["tool_calls"])
+            log_context.stacks.append({"component": "llm_handler", "data": data})
+        return resp
+
+    def _handle_response(self, response, tools_dict, messages, log_context):
+        is_structured_output = (
+            self.json_schema is not None
+            and hasattr(self.llm, "_supports_structured_output")
+            and self.llm._supports_structured_output()
+        )
+
+        if isinstance(response, str):
+            answer_data = {"answer": response}
+            if is_structured_output:
+                answer_data["structured"] = True
+                answer_data["schema"] = self.json_schema
+            yield answer_data
+            return
+        if hasattr(response, "message") and getattr(response.message, "content", None):
+            answer_data = {"answer": response.message.content}
+            if is_structured_output:
+                answer_data["structured"] = True
+                answer_data["schema"] = self.json_schema
+            yield answer_data
+            return
+        processed_response_gen = self._llm_handler(
+            response, tools_dict, messages, log_context, self.attachments
+        )
+
+        for event in processed_response_gen:
+            if isinstance(event, str):
+                answer_data = {"answer": event}
+                if is_structured_output:
+                    answer_data["structured"] = True
+                    answer_data["schema"] = self.json_schema
+                yield answer_data
+            elif hasattr(event, "message") and getattr(event.message, "content", None):
+                answer_data = {"answer": event.message.content}
+                if is_structured_output:
+                    answer_data["structured"] = True
+                    answer_data["schema"] = self.json_schema
+                yield answer_data
+            elif isinstance(event, dict) and "type" in event:
+                yield event

application/agents/classic_agent.py

@@ -0,0 +1,37 @@
+import logging
+from typing import Dict, Generator
+
+from application.agents.base import BaseAgent
+from application.logging import LogContext
+
+logger = logging.getLogger(__name__)
+
+
+class ClassicAgent(BaseAgent):
+    """A simplified agent with clear execution flow"""
+
+    def _gen_inner(
+        self, query: str, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        """Core generator function for ClassicAgent execution flow"""
+
+        tools_dict = (
+            self._get_user_tools(self.user)
+            if not self.user_api_key
+            else self._get_tools(self.user_api_key)
+        )
+        self._prepare_tools(tools_dict)
+
+        messages = self._build_messages(self.prompt, query)
+        llm_response = self._llm_gen(messages, log_context)
+
+        yield from self._handle_response(
+            llm_response, tools_dict, messages, log_context
+        )
+
+        yield {"sources": self.retrieved_docs}
+        yield {"tool_calls": self._get_truncated_tool_calls()}
+
+        log_context.stacks.append(
+            {"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}}
+        )

application/agents/react_agent.py

@@ -0,0 +1,238 @@
+import logging
+import os
+from typing import Any, Dict, Generator, List
+
+from application.agents.base import BaseAgent
+from application.logging import build_stack_data, LogContext
+
+logger = logging.getLogger(__name__)
+
+MAX_ITERATIONS_REASONING = 10
+
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+with open(
+    os.path.join(current_dir, "application/prompts", "react_planning_prompt.txt"), "r"
+) as f:
+    PLANNING_PROMPT_TEMPLATE = f.read()
+with open(
+    os.path.join(current_dir, "application/prompts", "react_final_prompt.txt"), "r"
+) as f:
+    FINAL_PROMPT_TEMPLATE = f.read()
+
+
+class ReActAgent(BaseAgent):
+    """
+    Research and Action (ReAct) Agent - Advanced reasoning agent with iterative planning.
+
+    Implements a think-act-observe loop for complex problem-solving:
+    1. Creates a strategic plan based on the query
+    2. Executes tools and gathers observations
+    3. Iteratively refines approach until satisfied
+    4. Synthesizes final answer from all observations
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.plan: str = ""
+        self.observations: List[str] = []
+
+    def _gen_inner(
+        self, query: str, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        """Execute ReAct reasoning loop with planning, action, and observation cycles"""
+
+        self._reset_state()
+
+        tools_dict = (
+            self._get_tools(self.user_api_key)
+            if self.user_api_key
+            else self._get_user_tools(self.user)
+        )
+        self._prepare_tools(tools_dict)
+
+        for iteration in range(1, MAX_ITERATIONS_REASONING + 1):
+            yield {"thought": f"Reasoning... (iteration {iteration})\n\n"}
+
+            yield from self._planning_phase(query, log_context)
+
+            if not self.plan:
+                logger.warning(
+                    f"ReActAgent: No plan generated in iteration {iteration}"
+                )
+                break
+            self.observations.append(f"Plan (iteration {iteration}): {self.plan}")
+
+            satisfied = yield from self._execution_phase(query, tools_dict, log_context)
+
+            if satisfied:
+                logger.info("ReActAgent: Goal satisfied, stopping reasoning loop")
+                break
+        yield from self._synthesis_phase(query, log_context)
+
+    def _reset_state(self):
+        """Reset agent state for new query"""
+        self.plan = ""
+        self.observations = []
+
+    def _planning_phase(
+        self, query: str, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        """Generate strategic plan for query"""
+        logger.info("ReActAgent: Creating plan...")
+
+        plan_prompt = self._build_planning_prompt(query)
+        messages = [{"role": "user", "content": plan_prompt}]
+
+        plan_stream = self.llm.gen_stream(
+            model=self.model_id,
+            messages=messages,
+            tools=self.tools if self.tools else None,
+        )
+
+        if log_context:
+            log_context.stacks.append(
+                {"component": "planning_llm", "data": build_stack_data(self.llm)}
+            )
+        plan_parts = []
+        for chunk in plan_stream:
+            content = self._extract_content(chunk)
+            if content:
+                plan_parts.append(content)
+                yield {"thought": content}
+        self.plan = "".join(plan_parts)
+
+    def _execution_phase(
+        self, query: str, tools_dict: Dict, log_context: LogContext
+    ) -> Generator[bool, None, None]:
+        """Execute plan with tool calls and observations"""
+        execution_prompt = self._build_execution_prompt(query)
+        messages = self._build_messages(execution_prompt, query)
+
+        llm_response = self._llm_gen(messages, log_context)
+        initial_content = self._extract_content(llm_response)
+
+        if initial_content:
+            self.observations.append(f"Initial response: {initial_content}")
+        processed_response = self._llm_handler(
+            llm_response, tools_dict, messages, log_context
+        )
+
+        for tool_call in self.tool_calls:
+            observation = (
+                f"Executed: {tool_call.get('tool_name', 'Unknown')} "
+                f"with args {tool_call.get('arguments', {})}. "
+                f"Result: {str(tool_call.get('result', ''))[:200]}"
+            )
+            self.observations.append(observation)
+        final_content = self._extract_content(processed_response)
+        if final_content:
+            self.observations.append(f"Response after tools: {final_content}")
+        if log_context:
+            log_context.stacks.append(
+                {
+                    "component": "agent_tool_calls",
+                    "data": {"tool_calls": self.tool_calls.copy()},
+                }
+            )
+        yield {"sources": self.retrieved_docs}
+        yield {"tool_calls": self._get_truncated_tool_calls()}
+
+        return "SATISFIED" in (final_content or "")
+
+    def _synthesis_phase(
+        self, query: str, log_context: LogContext
+    ) -> Generator[Dict, None, None]:
+        """Synthesize final answer from all observations"""
+        logger.info("ReActAgent: Generating final answer...")
+
+        final_prompt = self._build_final_answer_prompt(query)
+        messages = [{"role": "user", "content": final_prompt}]
+
+        final_stream = self.llm.gen_stream(
+            model=self.model_id, messages=messages, tools=None
+        )
+
+        if log_context:
+            log_context.stacks.append(
+                {"component": "final_answer_llm", "data": build_stack_data(self.llm)}
+            )
+        for chunk in final_stream:
+            content = self._extract_content(chunk)
+            if content:
+                yield {"answer": content}
+
+    def _build_planning_prompt(self, query: str) -> str:
+        """Build planning phase prompt"""
+        prompt = PLANNING_PROMPT_TEMPLATE.replace("{query}", query)
+        prompt = prompt.replace("{prompt}", self.prompt or "")
+        prompt = prompt.replace("{summaries}", "")
+        prompt = prompt.replace("{observations}", "\n".join(self.observations))
+        return prompt
+
+    def _build_execution_prompt(self, query: str) -> str:
+        """Build execution phase prompt with plan and observations"""
+        observations_str = "\n".join(self.observations)
+
+        if len(observations_str) > 20000:
+            observations_str = observations_str[:20000] + "\n...[truncated]"
+        return (
+            f"{self.prompt or ''}\n\n"
+            f"Follow this plan:\n{self.plan}\n\n"
+            f"Observations:\n{observations_str}\n\n"
+            f"If sufficient data exists to answer '{query}', respond with 'SATISFIED'. "
+            f"Otherwise, continue executing the plan."
+        )
+
+    def _build_final_answer_prompt(self, query: str) -> str:
+        """Build final synthesis prompt"""
+        observations_str = "\n".join(self.observations)
+
+        if len(observations_str) > 10000:
+            observations_str = observations_str[:10000] + "\n...[truncated]"
+            logger.warning("ReActAgent: Observations truncated for final answer")
+        return FINAL_PROMPT_TEMPLATE.format(query=query, observations=observations_str)
+
+    def _extract_content(self, response: Any) -> str:
+        """Extract text content from various LLM response formats"""
+        if not response:
+            return ""
+        collected = []
+
+        if isinstance(response, str):
+            return response
+        if hasattr(response, "message") and hasattr(response.message, "content"):
+            if response.message.content:
+                return response.message.content
+        if hasattr(response, "choices") and response.choices:
+            if hasattr(response.choices[0], "message"):
+                content = response.choices[0].message.content
+                if content:
+                    return content
+        if hasattr(response, "content") and isinstance(response.content, list):
+            if response.content and hasattr(response.content[0], "text"):
+                return response.content[0].text
+        try:
+            for chunk in response:
+                content_piece = ""
+
+                if hasattr(chunk, "choices") and chunk.choices:
+                    if hasattr(chunk.choices[0], "delta"):
+                        delta_content = chunk.choices[0].delta.content
+                        if delta_content:
+                            content_piece = delta_content
+                elif hasattr(chunk, "type") and chunk.type == "content_block_delta":
+                    if hasattr(chunk, "delta") and hasattr(chunk.delta, "text"):
+                        content_piece = chunk.delta.text
+                elif isinstance(chunk, str):
+                    content_piece = chunk
+                if content_piece:
+                    collected.append(content_piece)
+        except (TypeError, AttributeError):
+            logger.debug(
+                f"Response not iterable or unexpected format: {type(response)}"
+            )
+        except Exception as e:
+            logger.error(f"Error extracting content: {e}")
+        return "".join(collected)

application/agents/tools/api_tool.py

@@ -0,0 +1,72 @@
+import json
+
+import requests
+from application.agents.tools.base import Tool
+
+
+class APITool(Tool):
+    """
+    API Tool
+    A flexible tool for performing various API actions (e.g., sending messages, retrieving data) via custom user-specified APIs
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.url = config.get("url", "")
+        self.method = config.get("method", "GET")
+        self.headers = config.get("headers", {"Content-Type": "application/json"})
+        self.query_params = config.get("query_params", {})
+
+    def execute_action(self, action_name, **kwargs):
+        return self._make_api_call(
+            self.url, self.method, self.headers, self.query_params, kwargs
+        )
+
+    def _make_api_call(self, url, method, headers, query_params, body):
+        if query_params:
+            url = f"{url}?{requests.compat.urlencode(query_params)}"
+        # if isinstance(body, dict):
+        #     body = json.dumps(body)
+        try:
+            print(f"Making API call: {method} {url} with body: {body}")
+            if body == "{}":
+                body = None
+            response = requests.request(method, url, headers=headers, data=body)
+            response.raise_for_status()
+            content_type = response.headers.get(
+                "Content-Type", "application/json"
+            ).lower()
+            if "application/json" in content_type:
+                try:
+                    data = response.json()
+                except json.JSONDecodeError as e:
+                    print(f"Error decoding JSON: {e}.  Raw response: {response.text}")
+                    return {
+                        "status_code": response.status_code,
+                        "message": f"API call returned invalid JSON.  Error: {e}",
+                        "data": response.text,
+                    }
+            elif "text/" in content_type or "application/xml" in content_type:
+                data = response.text
+            elif not response.content:
+                data = None
+            else:
+                print(f"Unsupported content type: {content_type}")
+                data = response.content
+
+            return {
+                "status_code": response.status_code,
+                "data": data,
+                "message": "API call successful.",
+            }
+        except requests.exceptions.RequestException as e:
+            return {
+                "status_code": response.status_code if response else None,
+                "message": f"API call failed: {str(e)}",
+            }
+
+    def get_actions_metadata(self):
+        return []
+
+    def get_config_requirements(self):
+        return {}

application/agents/tools/base.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+
+
+class Tool(ABC):
+    @abstractmethod
+    def execute_action(self, action_name: str, **kwargs):
+        pass
+
+    @abstractmethod
+    def get_actions_metadata(self):
+        """
+        Returns a list of JSON objects describing the actions supported by the tool.
+        """
+        pass
+
+    @abstractmethod
+    def get_config_requirements(self):
+        """
+        Returns a dictionary describing the configuration requirements for the tool.
+        """
+        pass

application/agents/tools/brave.py

@@ -0,0 +1,182 @@
+import requests
+from application.agents.tools.base import Tool
+
+
+class BraveSearchTool(Tool):
+    """
+    Brave Search
+    A tool for performing web and image searches using the Brave Search API.
+    Requires an API key for authentication.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.token = config.get("token", "")
+        self.base_url = "https://api.search.brave.com/res/v1"
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "brave_web_search": self._web_search,
+            "brave_image_search": self._image_search,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _web_search(
+        self,
+        query,
+        country="ALL",
+        search_lang="en",
+        count=10,
+        offset=0,
+        safesearch="off",
+        freshness=None,
+        result_filter=None,
+        extra_snippets=False,
+        summary=False,
+    ):
+        """
+        Performs a web search using the Brave Search API.
+        """
+        print(f"Performing Brave web search for: {query}")
+
+        url = f"{self.base_url}/web/search"
+
+        params = {
+            "q": query,
+            "country": country,
+            "search_lang": search_lang,
+            "count": min(count, 20),
+            "offset": min(offset, 9),
+            "safesearch": safesearch,
+        }
+
+        if freshness:
+            params["freshness"] = freshness
+        if result_filter:
+            params["result_filter"] = result_filter
+        if extra_snippets:
+            params["extra_snippets"] = 1
+        if summary:
+            params["summary"] = 1
+        headers = {
+            "Accept": "application/json",
+            "Accept-Encoding": "gzip",
+            "X-Subscription-Token": self.token,
+        }
+
+        response = requests.get(url, params=params, headers=headers)
+
+        if response.status_code == 200:
+            return {
+                "status_code": response.status_code,
+                "results": response.json(),
+                "message": "Search completed successfully.",
+            }
+        else:
+            return {
+                "status_code": response.status_code,
+                "message": f"Search failed with status code: {response.status_code}.",
+            }
+
+    def _image_search(
+        self,
+        query,
+        country="ALL",
+        search_lang="en",
+        count=5,
+        safesearch="off",
+        spellcheck=False,
+    ):
+        """
+        Performs an image search using the Brave Search API.
+        """
+        print(f"Performing Brave image search for: {query}")
+
+        url = f"{self.base_url}/images/search"
+
+        params = {
+            "q": query,
+            "country": country,
+            "search_lang": search_lang,
+            "count": min(count, 100),  # API max is 100
+            "safesearch": safesearch,
+            "spellcheck": 1 if spellcheck else 0,
+        }
+
+        headers = {
+            "Accept": "application/json",
+            "Accept-Encoding": "gzip",
+            "X-Subscription-Token": self.token,
+        }
+
+        response = requests.get(url, params=params, headers=headers)
+
+        if response.status_code == 200:
+            return {
+                "status_code": response.status_code,
+                "results": response.json(),
+                "message": "Image search completed successfully.",
+            }
+        else:
+            return {
+                "status_code": response.status_code,
+                "message": f"Image search failed with status code: {response.status_code}.",
+            }
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "brave_web_search",
+                "description": "Perform a web search using Brave Search",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "The search query (max 400 characters, 50 words)",
+                        },
+                        "search_lang": {
+                            "type": "string",
+                            "description": "The search language preference (default: en)",
+                        },
+                        "freshness": {
+                            "type": "string",
+                            "description": "Time filter for results (pd: last 24h, pw: last week, pm: last month, py: last year)",
+                        },
+                    },
+                    "required": ["query"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "brave_image_search",
+                "description": "Perform an image search using Brave Search",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "The search query (max 400 characters, 50 words)",
+                        },
+                        "count": {
+                            "type": "integer",
+                            "description": "Number of results to return (max 100, default: 5)",
+                        },
+                    },
+                    "required": ["query"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {
+                "type": "string",
+                "description": "Brave Search API key for authentication",
+            },
+        }

application/agents/tools/cryptoprice.py

@@ -0,0 +1,76 @@
+import requests
+from application.agents.tools.base import Tool
+
+
+class CryptoPriceTool(Tool):
+    """
+    CryptoPrice
+    A tool for retrieving cryptocurrency prices using the CryptoCompare public API
+    """
+
+    def __init__(self, config):
+        self.config = config
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {"cryptoprice_get": self._get_price}
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _get_price(self, symbol, currency):
+        """
+        Fetches the current price of a given cryptocurrency symbol in the specified currency.
+        Example:
+            symbol = "BTC"
+            currency = "USD"
+            returns price in USD.
+        """
+        url = f"https://min-api.cryptocompare.com/data/price?fsym={symbol.upper()}&tsyms={currency.upper()}"
+        response = requests.get(url)
+        if response.status_code == 200:
+            data = response.json()
+            if currency.upper() in data:
+                return {
+                    "status_code": response.status_code,
+                    "price": data[currency.upper()],
+                    "message": f"Price of {symbol.upper()} in {currency.upper()} retrieved successfully.",
+                }
+            else:
+                return {
+                    "status_code": response.status_code,
+                    "message": f"Couldn't find price for {symbol.upper()} in {currency.upper()}.",
+                }
+        else:
+            return {
+                "status_code": response.status_code,
+                "message": "Failed to retrieve price.",
+            }
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "cryptoprice_get",
+                "description": "Retrieve the price of a specified cryptocurrency in a given currency",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "symbol": {
+                            "type": "string",
+                            "description": "The cryptocurrency symbol (e.g. BTC)",
+                        },
+                        "currency": {
+                            "type": "string",
+                            "description": "The currency in which you want the price (e.g. USD)",
+                        },
+                    },
+                    "required": ["symbol", "currency"],
+                    "additionalProperties": False,
+                },
+            }
+        ]
+
+    def get_config_requirements(self):
+        # No specific configuration needed for this tool as it just queries a public endpoint
+        return {}

application/agents/tools/duckduckgo.py

@@ -0,0 +1,114 @@
+from application.agents.tools.base import Tool
+from duckduckgo_search import DDGS
+
+
+class DuckDuckGoSearchTool(Tool):
+    """
+    DuckDuckGo Search
+    A tool for performing web and image searches using DuckDuckGo.
+    """
+
+    def __init__(self, config):
+        self.config = config
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "ddg_web_search": self._web_search,
+            "ddg_image_search": self._image_search,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _web_search(
+        self,
+        query,
+        max_results=5,
+    ):
+        print(f"Performing DuckDuckGo web search for: {query}")
+
+        try:
+            results = DDGS().text(
+                query,
+                max_results=max_results,
+            )
+
+            return {
+                "status_code": 200,
+                "results": results,
+                "message": "Web search completed successfully.",
+            }
+        except Exception as e:
+            return {
+                "status_code": 500,
+                "message": f"Web search failed: {str(e)}",
+            }
+
+    def _image_search(
+        self,
+        query,
+        max_results=5,
+    ):
+        print(f"Performing DuckDuckGo image search for: {query}")
+
+        try:
+            results = DDGS().images(
+                keywords=query,
+                max_results=max_results,
+            )
+
+            return {
+                "status_code": 200,
+                "results": results,
+                "message": "Image search completed successfully.",
+            }
+        except Exception as e:
+            return {
+                "status_code": 500,
+                "message": f"Image search failed: {str(e)}",
+            }
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "ddg_web_search",
+                "description": "Perform a web search using DuckDuckGo.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "Search query",
+                        },
+                        "max_results": {
+                            "type": "integer",
+                            "description": "Number of results to return (default: 5)",
+                        },
+                    },
+                    "required": ["query"],
+                },
+            },
+            {
+                "name": "ddg_image_search",
+                "description": "Perform an image search using DuckDuckGo.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "Search query",
+                        },
+                        "max_results": {
+                            "type": "integer",
+                            "description": "Number of results to return (default: 5, max: 50)",
+                        },
+                    },
+                    "required": ["query"],
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {}

application/agents/tools/mcp_tool.py

@@ -0,0 +1,861 @@
+import asyncio
+import base64
+import json
+import logging
+import time
+from typing import Any, Dict, List, Optional
+from urllib.parse import parse_qs, urlparse
+
+from application.agents.tools.base import Tool
+from application.api.user.tasks import mcp_oauth_status_task, mcp_oauth_task
+from application.cache import get_redis_instance
+
+from application.core.mongo_db import MongoDB
+
+from application.core.settings import settings
+
+from application.security.encryption import decrypt_credentials
+from fastmcp import Client
+from fastmcp.client.auth import BearerAuth
+from fastmcp.client.transports import (
+    SSETransport,
+    StdioTransport,
+    StreamableHttpTransport,
+)
+from mcp.client.auth import OAuthClientProvider, TokenStorage
+from mcp.shared.auth import OAuthClientInformationFull, OAuthClientMetadata, OAuthToken
+
+from pydantic import AnyHttpUrl, ValidationError
+from redis import Redis
+
+mongo = MongoDB.get_client()
+db = mongo[settings.MONGO_DB_NAME]
+
+_mcp_clients_cache = {}
+
+
+class MCPTool(Tool):
+    """
+    MCP Tool
+    Connect to remote Model Context Protocol (MCP) servers to access dynamic tools and resources.
+    """
+
+    def __init__(self, config: Dict[str, Any], user_id: Optional[str] = None):
+        """
+        Initialize the MCP Tool with configuration.
+
+        Args:
+            config: Dictionary containing MCP server configuration:
+                - server_url: URL of the remote MCP server
+                - transport_type: Transport type (auto, sse, http, stdio)
+                - auth_type: Type of authentication (bearer, oauth, api_key, basic, none)
+                - encrypted_credentials: Encrypted credentials (if available)
+                - timeout: Request timeout in seconds (default: 30)
+                - headers: Custom headers for requests
+                - command: Command for STDIO transport
+                - args: Arguments for STDIO transport
+                - oauth_scopes: OAuth scopes for oauth auth type
+                - oauth_client_name: OAuth client name for oauth auth type
+            user_id: User ID for decrypting credentials (required if encrypted_credentials exist)
+        """
+        self.config = config
+        self.user_id = user_id
+        self.server_url = config.get("server_url", "")
+        self.transport_type = config.get("transport_type", "auto")
+        self.auth_type = config.get("auth_type", "none")
+        self.timeout = config.get("timeout", 30)
+        self.custom_headers = config.get("headers", {})
+
+        self.auth_credentials = {}
+        if config.get("encrypted_credentials") and user_id:
+            self.auth_credentials = decrypt_credentials(
+                config["encrypted_credentials"], user_id
+            )
+        else:
+            self.auth_credentials = config.get("auth_credentials", {})
+        self.oauth_scopes = config.get("oauth_scopes", [])
+        self.oauth_task_id = config.get("oauth_task_id", None)
+        self.oauth_client_name = config.get("oauth_client_name", "DocsGPT-MCP")
+        self.redirect_uri = f"{settings.API_URL}/api/mcp_server/callback"
+
+        self.available_tools = []
+        self._cache_key = self._generate_cache_key()
+        self._client = None
+
+        # Only validate and setup if server_url is provided and not OAuth
+
+        if self.server_url and self.auth_type != "oauth":
+            self._setup_client()
+
+    def _generate_cache_key(self) -> str:
+        """Generate a unique cache key for this MCP server configuration."""
+        auth_key = ""
+        if self.auth_type == "oauth":
+            scopes_str = ",".join(self.oauth_scopes) if self.oauth_scopes else "none"
+            auth_key = f"oauth:{self.oauth_client_name}:{scopes_str}"
+        elif self.auth_type in ["bearer"]:
+            token = self.auth_credentials.get(
+                "bearer_token", ""
+            ) or self.auth_credentials.get("access_token", "")
+            auth_key = f"bearer:{token[:10]}..." if token else "bearer:none"
+        elif self.auth_type == "api_key":
+            api_key = self.auth_credentials.get("api_key", "")
+            auth_key = f"apikey:{api_key[:10]}..." if api_key else "apikey:none"
+        elif self.auth_type == "basic":
+            username = self.auth_credentials.get("username", "")
+            auth_key = f"basic:{username}"
+        else:
+            auth_key = "none"
+        return f"{self.server_url}#{self.transport_type}#{auth_key}"
+
+    def _setup_client(self):
+        """Setup FastMCP client with proper transport and authentication."""
+        global _mcp_clients_cache
+        if self._cache_key in _mcp_clients_cache:
+            cached_data = _mcp_clients_cache[self._cache_key]
+            if time.time() - cached_data["created_at"] < 1800:
+                self._client = cached_data["client"]
+                return
+            else:
+                del _mcp_clients_cache[self._cache_key]
+        transport = self._create_transport()
+        auth = None
+
+        if self.auth_type == "oauth":
+            redis_client = get_redis_instance()
+            auth = DocsGPTOAuth(
+                mcp_url=self.server_url,
+                scopes=self.oauth_scopes,
+                redis_client=redis_client,
+                redirect_uri=self.redirect_uri,
+                task_id=self.oauth_task_id,
+                db=db,
+                user_id=self.user_id,
+            )
+        elif self.auth_type == "bearer":
+            token = self.auth_credentials.get(
+                "bearer_token", ""
+            ) or self.auth_credentials.get("access_token", "")
+            if token:
+                auth = BearerAuth(token)
+        self._client = Client(transport, auth=auth)
+        _mcp_clients_cache[self._cache_key] = {
+            "client": self._client,
+            "created_at": time.time(),
+        }
+
+    def _create_transport(self):
+        """Create appropriate transport based on configuration."""
+        headers = {"Content-Type": "application/json", "User-Agent": "DocsGPT-MCP/1.0"}
+        headers.update(self.custom_headers)
+
+        if self.auth_type == "api_key":
+            api_key = self.auth_credentials.get("api_key", "")
+            header_name = self.auth_credentials.get("api_key_header", "X-API-Key")
+            if api_key:
+                headers[header_name] = api_key
+        elif self.auth_type == "basic":
+            username = self.auth_credentials.get("username", "")
+            password = self.auth_credentials.get("password", "")
+            if username and password:
+                credentials = base64.b64encode(
+                    f"{username}:{password}".encode()
+                ).decode()
+                headers["Authorization"] = f"Basic {credentials}"
+        if self.transport_type == "auto":
+            if "sse" in self.server_url.lower() or self.server_url.endswith("/sse"):
+                transport_type = "sse"
+            else:
+                transport_type = "http"
+        else:
+            transport_type = self.transport_type
+        if transport_type == "sse":
+            headers.update({"Accept": "text/event-stream", "Cache-Control": "no-cache"})
+            return SSETransport(url=self.server_url, headers=headers)
+        elif transport_type == "http":
+            return StreamableHttpTransport(url=self.server_url, headers=headers)
+        elif transport_type == "stdio":
+            command = self.config.get("command", "python")
+            args = self.config.get("args", [])
+            env = self.auth_credentials if self.auth_credentials else None
+            return StdioTransport(command=command, args=args, env=env)
+        else:
+            return StreamableHttpTransport(url=self.server_url, headers=headers)
+
+    def _format_tools(self, tools_response) -> List[Dict]:
+        """Format tools response to match expected format."""
+        if hasattr(tools_response, "tools"):
+            tools = tools_response.tools
+        elif isinstance(tools_response, list):
+            tools = tools_response
+        else:
+            tools = []
+        tools_dict = []
+        for tool in tools:
+            if hasattr(tool, "name"):
+                tool_dict = {
+                    "name": tool.name,
+                    "description": tool.description,
+                }
+                if hasattr(tool, "inputSchema"):
+                    tool_dict["inputSchema"] = tool.inputSchema
+                tools_dict.append(tool_dict)
+            elif isinstance(tool, dict):
+                tools_dict.append(tool)
+            else:
+                if hasattr(tool, "model_dump"):
+                    tools_dict.append(tool.model_dump())
+                else:
+                    tools_dict.append({"name": str(tool), "description": ""})
+        return tools_dict
+
+    async def _execute_with_client(self, operation: str, *args, **kwargs):
+        """Execute operation with FastMCP client."""
+        if not self._client:
+            raise Exception("FastMCP client not initialized")
+        async with self._client:
+            if operation == "ping":
+                return await self._client.ping()
+            elif operation == "list_tools":
+                tools_response = await self._client.list_tools()
+                self.available_tools = self._format_tools(tools_response)
+                return self.available_tools
+            elif operation == "call_tool":
+                tool_name = args[0]
+                tool_args = kwargs
+                return await self._client.call_tool(tool_name, tool_args)
+            elif operation == "list_resources":
+                return await self._client.list_resources()
+            elif operation == "list_prompts":
+                return await self._client.list_prompts()
+            else:
+                raise Exception(f"Unknown operation: {operation}")
+
+    def _run_async_operation(self, operation: str, *args, **kwargs):
+        """Run async operation in sync context."""
+        try:
+            try:
+                loop = asyncio.get_running_loop()
+                import concurrent.futures
+
+                def run_in_thread():
+                    new_loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(new_loop)
+                    try:
+                        return new_loop.run_until_complete(
+                            self._execute_with_client(operation, *args, **kwargs)
+                        )
+                    finally:
+                        new_loop.close()
+
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    future = executor.submit(run_in_thread)
+                    return future.result(timeout=self.timeout)
+            except RuntimeError:
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+                try:
+                    return loop.run_until_complete(
+                        self._execute_with_client(operation, *args, **kwargs)
+                    )
+                finally:
+                    loop.close()
+        except Exception as e:
+            print(f"Error occurred while running async operation: {e}")
+            raise
+
+    def discover_tools(self) -> List[Dict]:
+        """
+        Discover available tools from the MCP server using FastMCP.
+
+        Returns:
+            List of tool definitions from the server
+        """
+        if not self.server_url:
+            return []
+        if not self._client:
+            self._setup_client()
+        try:
+            tools = self._run_async_operation("list_tools")
+            self.available_tools = tools
+            return self.available_tools
+        except Exception as e:
+            raise Exception(f"Failed to discover tools from MCP server: {str(e)}")
+
+    def execute_action(self, action_name: str, **kwargs) -> Any:
+        """
+        Execute an action on the remote MCP server using FastMCP.
+
+        Args:
+            action_name: Name of the action to execute
+            **kwargs: Parameters for the action
+
+        Returns:
+            Result from the MCP server
+        """
+        if not self.server_url:
+            raise Exception("No MCP server configured")
+        if not self._client:
+            self._setup_client()
+        cleaned_kwargs = {}
+        for key, value in kwargs.items():
+            if value == "" or value is None:
+                continue
+            cleaned_kwargs[key] = value
+        try:
+            result = self._run_async_operation(
+                "call_tool", action_name, **cleaned_kwargs
+            )
+            return self._format_result(result)
+        except Exception as e:
+            raise Exception(f"Failed to execute action '{action_name}': {str(e)}")
+
+    def _format_result(self, result) -> Dict:
+        """Format FastMCP result to match expected format."""
+        if hasattr(result, "content"):
+            content_list = []
+            for content_item in result.content:
+                if hasattr(content_item, "text"):
+                    content_list.append({"type": "text", "text": content_item.text})
+                elif hasattr(content_item, "data"):
+                    content_list.append({"type": "data", "data": content_item.data})
+                else:
+                    content_list.append(
+                        {"type": "unknown", "content": str(content_item)}
+                    )
+            return {
+                "content": content_list,
+                "isError": getattr(result, "isError", False),
+            }
+        else:
+            return result
+
+    def test_connection(self) -> Dict:
+        """
+        Test the connection to the MCP server and validate functionality.
+
+        Returns:
+            Dictionary with connection test results including tool count
+        """
+        if not self.server_url:
+            return {
+                "success": False,
+                "message": "No MCP server URL configured",
+                "tools_count": 0,
+                "transport_type": self.transport_type,
+                "auth_type": self.auth_type,
+                "error_type": "ConfigurationError",
+            }
+        if not self._client:
+            self._setup_client()
+        try:
+            if self.auth_type == "oauth":
+                return self._test_oauth_connection()
+            else:
+                return self._test_regular_connection()
+        except Exception as e:
+            return {
+                "success": False,
+                "message": f"Connection failed: {str(e)}",
+                "tools_count": 0,
+                "transport_type": self.transport_type,
+                "auth_type": self.auth_type,
+                "error_type": type(e).__name__,
+            }
+
+    def _test_regular_connection(self) -> Dict:
+        """Test connection for non-OAuth auth types."""
+        try:
+            self._run_async_operation("ping")
+            ping_success = True
+        except Exception:
+            ping_success = False
+        tools = self.discover_tools()
+
+        message = f"Successfully connected to MCP server. Found {len(tools)} tools."
+        if not ping_success:
+            message += " (Ping not supported, but tool discovery worked)"
+        return {
+            "success": True,
+            "message": message,
+            "tools_count": len(tools),
+            "transport_type": self.transport_type,
+            "auth_type": self.auth_type,
+            "ping_supported": ping_success,
+            "tools": [tool.get("name", "unknown") for tool in tools],
+        }
+
+    def _test_oauth_connection(self) -> Dict:
+        """Test connection for OAuth auth type with proper async handling."""
+        try:
+            task = mcp_oauth_task.delay(config=self.config, user=self.user_id)
+            if not task:
+                raise Exception("Failed to start OAuth authentication")
+            return {
+                "success": True,
+                "requires_oauth": True,
+                "task_id": task.id,
+                "status": "pending",
+                "message": "OAuth flow started",
+            }
+        except Exception as e:
+            return {
+                "success": False,
+                "message": f"OAuth connection failed: {str(e)}",
+                "tools_count": 0,
+                "transport_type": self.transport_type,
+                "auth_type": self.auth_type,
+                "error_type": type(e).__name__,
+            }
+
+    def get_actions_metadata(self) -> List[Dict]:
+        """
+        Get metadata for all available actions.
+
+        Returns:
+            List of action metadata dictionaries
+        """
+        actions = []
+        for tool in self.available_tools:
+            input_schema = (
+                tool.get("inputSchema")
+                or tool.get("input_schema")
+                or tool.get("schema")
+                or tool.get("parameters")
+            )
+
+            parameters_schema = {
+                "type": "object",
+                "properties": {},
+                "required": [],
+            }
+
+            if input_schema:
+                if isinstance(input_schema, dict):
+                    if "properties" in input_schema:
+                        parameters_schema = {
+                            "type": input_schema.get("type", "object"),
+                            "properties": input_schema.get("properties", {}),
+                            "required": input_schema.get("required", []),
+                        }
+
+                        for key in ["additionalProperties", "description"]:
+                            if key in input_schema:
+                                parameters_schema[key] = input_schema[key]
+                    else:
+                        parameters_schema["properties"] = input_schema
+            action = {
+                "name": tool.get("name", ""),
+                "description": tool.get("description", ""),
+                "parameters": parameters_schema,
+            }
+            actions.append(action)
+        return actions
+
+    def get_config_requirements(self) -> Dict:
+        """Get configuration requirements for the MCP tool."""
+        return {
+            "server_url": {
+                "type": "string",
+                "description": "URL of the remote MCP server (e.g., https://api.example.com/mcp or https://docs.mcp.cloudflare.com/sse)",
+                "required": True,
+            },
+            "transport_type": {
+                "type": "string",
+                "description": "Transport type for connection",
+                "enum": ["auto", "sse", "http", "stdio"],
+                "default": "auto",
+                "required": False,
+                "help": {
+                    "auto": "Automatically detect best transport",
+                    "sse": "Server-Sent Events (for real-time streaming)",
+                    "http": "HTTP streaming (recommended for production)",
+                    "stdio": "Standard I/O (for local servers)",
+                },
+            },
+            "auth_type": {
+                "type": "string",
+                "description": "Authentication type",
+                "enum": ["none", "bearer", "oauth", "api_key", "basic"],
+                "default": "none",
+                "required": True,
+                "help": {
+                    "none": "No authentication",
+                    "bearer": "Bearer token authentication",
+                    "oauth": "OAuth 2.1 authentication (with frontend integration)",
+                    "api_key": "API key authentication",
+                    "basic": "Basic authentication",
+                },
+            },
+            "auth_credentials": {
+                "type": "object",
+                "description": "Authentication credentials (varies by auth_type)",
+                "required": False,
+                "properties": {
+                    "bearer_token": {
+                        "type": "string",
+                        "description": "Bearer token for bearer auth",
+                    },
+                    "access_token": {
+                        "type": "string",
+                        "description": "Access token for OAuth (if pre-obtained)",
+                    },
+                    "api_key": {
+                        "type": "string",
+                        "description": "API key for api_key auth",
+                    },
+                    "api_key_header": {
+                        "type": "string",
+                        "description": "Header name for API key (default: X-API-Key)",
+                    },
+                    "username": {
+                        "type": "string",
+                        "description": "Username for basic auth",
+                    },
+                    "password": {
+                        "type": "string",
+                        "description": "Password for basic auth",
+                    },
+                },
+            },
+            "oauth_scopes": {
+                "type": "array",
+                "description": "OAuth scopes to request (for oauth auth_type)",
+                "items": {"type": "string"},
+                "required": False,
+                "default": [],
+            },
+            "oauth_client_name": {
+                "type": "string",
+                "description": "Client name for OAuth registration (for oauth auth_type)",
+                "default": "DocsGPT-MCP",
+                "required": False,
+            },
+            "headers": {
+                "type": "object",
+                "description": "Custom headers to send with requests",
+                "required": False,
+            },
+            "timeout": {
+                "type": "integer",
+                "description": "Request timeout in seconds",
+                "default": 30,
+                "minimum": 1,
+                "maximum": 300,
+                "required": False,
+            },
+            "command": {
+                "type": "string",
+                "description": "Command to run for STDIO transport (e.g., 'python')",
+                "required": False,
+            },
+            "args": {
+                "type": "array",
+                "description": "Arguments for STDIO command",
+                "items": {"type": "string"},
+                "required": False,
+            },
+        }
+
+
+class DocsGPTOAuth(OAuthClientProvider):
+    """
+    Custom OAuth handler for DocsGPT that uses frontend redirect instead of browser.
+    """
+
+    def __init__(
+        self,
+        mcp_url: str,
+        redirect_uri: str,
+        redis_client: Redis | None = None,
+        redis_prefix: str = "mcp_oauth:",
+        task_id: str = None,
+        scopes: str | list[str] | None = None,
+        client_name: str = "DocsGPT-MCP",
+        user_id=None,
+        db=None,
+        additional_client_metadata: dict[str, Any] | None = None,
+    ):
+        """
+        Initialize custom OAuth client provider for DocsGPT.
+
+        Args:
+            mcp_url: Full URL to the MCP endpoint
+            redirect_uri: Custom redirect URI for DocsGPT frontend
+            redis_client: Redis client for storing auth state
+            redis_prefix: Prefix for Redis keys
+            task_id: Task ID for tracking auth status
+            scopes: OAuth scopes to request
+            client_name: Name for this client during registration
+            user_id: User ID for token storage
+            db: Database instance for token storage
+            additional_client_metadata: Extra fields for OAuthClientMetadata
+        """
+
+        self.redirect_uri = redirect_uri
+        self.redis_client = redis_client
+        self.redis_prefix = redis_prefix
+        self.task_id = task_id
+        self.user_id = user_id
+        self.db = db
+
+        parsed_url = urlparse(mcp_url)
+        self.server_base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+
+        if isinstance(scopes, list):
+            scopes = " ".join(scopes)
+        client_metadata = OAuthClientMetadata(
+            client_name=client_name,
+            redirect_uris=[AnyHttpUrl(redirect_uri)],
+            grant_types=["authorization_code", "refresh_token"],
+            response_types=["code"],
+            scope=scopes,
+            **(additional_client_metadata or {}),
+        )
+
+        storage = DBTokenStorage(
+            server_url=self.server_base_url, user_id=self.user_id, db_client=self.db
+        )
+
+        super().__init__(
+            server_url=self.server_base_url,
+            client_metadata=client_metadata,
+            storage=storage,
+            redirect_handler=self.redirect_handler,
+            callback_handler=self.callback_handler,
+        )
+
+        self.auth_url = None
+        self.extracted_state = None
+
+    def _process_auth_url(self, authorization_url: str) -> tuple[str, str]:
+        """Process authorization URL to extract state"""
+        try:
+            parsed_url = urlparse(authorization_url)
+            query_params = parse_qs(parsed_url.query)
+
+            state_params = query_params.get("state", [])
+            if state_params:
+                state = state_params[0]
+            else:
+                raise ValueError("No state in auth URL")
+            return authorization_url, state
+        except Exception as e:
+            raise Exception(f"Failed to process auth URL: {e}")
+
+    async def redirect_handler(self, authorization_url: str) -> None:
+        """Store auth URL and state in Redis for frontend to use."""
+        auth_url, state = self._process_auth_url(authorization_url)
+        logging.info(
+            "[DocsGPTOAuth] Processed auth_url: %s, state: %s", auth_url, state
+        )
+        self.auth_url = auth_url
+        self.extracted_state = state
+
+        if self.redis_client and self.extracted_state:
+            key = f"{self.redis_prefix}auth_url:{self.extracted_state}"
+            self.redis_client.setex(key, 600, auth_url)
+            logging.info("[DocsGPTOAuth] Stored auth_url in Redis: %s", key)
+
+            if self.task_id:
+                status_key = f"mcp_oauth_status:{self.task_id}"
+                status_data = {
+                    "status": "requires_redirect",
+                    "message": "OAuth authorization required",
+                    "authorization_url": self.auth_url,
+                    "state": self.extracted_state,
+                    "requires_oauth": True,
+                    "task_id": self.task_id,
+                }
+                self.redis_client.setex(status_key, 600, json.dumps(status_data))
+
+    async def callback_handler(self) -> tuple[str, str | None]:
+        """Wait for auth code from Redis using the state value."""
+        if not self.redis_client or not self.extracted_state:
+            raise Exception("Redis client or state not configured for OAuth")
+        poll_interval = 1
+        max_wait_time = 300
+        code_key = f"{self.redis_prefix}code:{self.extracted_state}"
+
+        if self.task_id:
+            status_key = f"mcp_oauth_status:{self.task_id}"
+            status_data = {
+                "status": "awaiting_callback",
+                "message": "Waiting for OAuth callback...",
+                "authorization_url": self.auth_url,
+                "state": self.extracted_state,
+                "requires_oauth": True,
+                "task_id": self.task_id,
+            }
+            self.redis_client.setex(status_key, 600, json.dumps(status_data))
+        start_time = time.time()
+        while time.time() - start_time < max_wait_time:
+            code_data = self.redis_client.get(code_key)
+            if code_data:
+                code = code_data.decode()
+                returned_state = self.extracted_state
+
+                self.redis_client.delete(code_key)
+                self.redis_client.delete(
+                    f"{self.redis_prefix}auth_url:{self.extracted_state}"
+                )
+                self.redis_client.delete(
+                    f"{self.redis_prefix}state:{self.extracted_state}"
+                )
+
+                if self.task_id:
+                    status_data = {
+                        "status": "callback_received",
+                        "message": "OAuth callback received, completing authentication...",
+                        "task_id": self.task_id,
+                    }
+                    self.redis_client.setex(status_key, 600, json.dumps(status_data))
+                return code, returned_state
+            error_key = f"{self.redis_prefix}error:{self.extracted_state}"
+            error_data = self.redis_client.get(error_key)
+            if error_data:
+                error_msg = error_data.decode()
+                self.redis_client.delete(error_key)
+                self.redis_client.delete(
+                    f"{self.redis_prefix}auth_url:{self.extracted_state}"
+                )
+                self.redis_client.delete(
+                    f"{self.redis_prefix}state:{self.extracted_state}"
+                )
+                raise Exception(f"OAuth error: {error_msg}")
+            await asyncio.sleep(poll_interval)
+        self.redis_client.delete(f"{self.redis_prefix}auth_url:{self.extracted_state}")
+        self.redis_client.delete(f"{self.redis_prefix}state:{self.extracted_state}")
+        raise Exception("OAuth callback timeout: no code received within 5 minutes")
+
+
+class DBTokenStorage(TokenStorage):
+    def __init__(self, server_url: str, user_id: str, db_client):
+        self.server_url = server_url
+        self.user_id = user_id
+        self.db_client = db_client
+        self.collection = db_client["connector_sessions"]
+
+    @staticmethod
+    def get_base_url(url: str) -> str:
+        parsed = urlparse(url)
+        return f"{parsed.scheme}://{parsed.netloc}"
+
+    def get_db_key(self) -> dict:
+        return {
+            "server_url": self.get_base_url(self.server_url),
+            "user_id": self.user_id,
+        }
+
+    async def get_tokens(self) -> OAuthToken | None:
+        doc = await asyncio.to_thread(self.collection.find_one, self.get_db_key())
+        if not doc or "tokens" not in doc:
+            return None
+        try:
+            tokens = OAuthToken.model_validate(doc["tokens"])
+            return tokens
+        except ValidationError as e:
+            logging.error(f"Could not load tokens: {e}")
+            return None
+
+    async def set_tokens(self, tokens: OAuthToken) -> None:
+        await asyncio.to_thread(
+            self.collection.update_one,
+            self.get_db_key(),
+            {"$set": {"tokens": tokens.model_dump()}},
+            True,
+        )
+        logging.info(f"Saved tokens for {self.get_base_url(self.server_url)}")
+
+    async def get_client_info(self) -> OAuthClientInformationFull | None:
+        doc = await asyncio.to_thread(self.collection.find_one, self.get_db_key())
+        if not doc or "client_info" not in doc:
+            return None
+        try:
+            client_info = OAuthClientInformationFull.model_validate(doc["client_info"])
+            tokens = await self.get_tokens()
+            if tokens is None:
+                logging.debug(
+                    "No tokens found, clearing client info to force fresh registration."
+                )
+                await asyncio.to_thread(
+                    self.collection.update_one,
+                    self.get_db_key(),
+                    {"$unset": {"client_info": ""}},
+                )
+                return None
+            return client_info
+        except ValidationError as e:
+            logging.error(f"Could not load client info: {e}")
+            return None
+
+    def _serialize_client_info(self, info: dict) -> dict:
+        if "redirect_uris" in info and isinstance(info["redirect_uris"], list):
+            info["redirect_uris"] = [str(u) for u in info["redirect_uris"]]
+        return info
+
+    async def set_client_info(self, client_info: OAuthClientInformationFull) -> None:
+        serialized_info = self._serialize_client_info(client_info.model_dump())
+        await asyncio.to_thread(
+            self.collection.update_one,
+            self.get_db_key(),
+            {"$set": {"client_info": serialized_info}},
+            True,
+        )
+        logging.info(f"Saved client info for {self.get_base_url(self.server_url)}")
+
+    async def clear(self) -> None:
+        await asyncio.to_thread(self.collection.delete_one, self.get_db_key())
+        logging.info(f"Cleared OAuth cache for {self.get_base_url(self.server_url)}")
+
+    @classmethod
+    async def clear_all(cls, db_client) -> None:
+        collection = db_client["connector_sessions"]
+        await asyncio.to_thread(collection.delete_many, {})
+        logging.info("Cleared all OAuth client cache data.")
+
+
+class MCPOAuthManager:
+    """Manager for handling MCP OAuth callbacks."""
+
+    def __init__(self, redis_client: Redis | None, redis_prefix: str = "mcp_oauth:"):
+        self.redis_client = redis_client
+        self.redis_prefix = redis_prefix
+
+    def handle_oauth_callback(
+        self, state: str, code: str, error: Optional[str] = None
+    ) -> bool:
+        """
+        Handle OAuth callback from provider.
+
+        Args:
+            state: The state parameter from OAuth callback
+            code: The authorization code from OAuth callback
+            error: Error message if OAuth failed
+
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            if not self.redis_client or not state:
+                raise Exception("Redis client or state not provided")
+            if error:
+                error_key = f"{self.redis_prefix}error:{state}"
+                self.redis_client.setex(error_key, 300, error)
+                raise Exception(f"OAuth error received: {error}")
+            code_key = f"{self.redis_prefix}code:{state}"
+            self.redis_client.setex(code_key, 300, code)
+
+            state_key = f"{self.redis_prefix}state:{state}"
+            self.redis_client.setex(state_key, 300, "completed")
+
+            return True
+        except Exception as e:
+            logging.error(f"Error handling OAuth callback: {e}")
+            return False
+
+    def get_oauth_status(self, task_id: str) -> Dict[str, Any]:
+        """Get current status of OAuth flow using provided task_id."""
+        if not task_id:
+            return {"status": "not_started", "message": "OAuth flow not started"}
+        return mcp_oauth_status_task(task_id)

application/agents/tools/memory.py

@@ -0,0 +1,546 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+import re
+import uuid
+
+from .base import Tool
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+
+
+class MemoryTool(Tool):
+    """Memory
+
+    Stores and retrieves information across conversations through a memory file directory.
+    """
+
+    def __init__(self, tool_config: Optional[Dict[str, Any]] = None, user_id: Optional[str] = None) -> None:
+        """Initialize the tool.
+
+        Args:
+            tool_config: Optional tool configuration. Should include:
+                - tool_id: Unique identifier for this memory tool instance (from user_tools._id)
+                           This ensures each user's tool configuration has isolated memories
+            user_id: The authenticated user's id (should come from decoded_token["sub"]).
+        """
+        self.user_id: Optional[str] = user_id
+
+        # Get tool_id from configuration (passed from user_tools._id in production)
+        # In production, tool_id is the MongoDB ObjectId string from user_tools collection
+        if tool_config and "tool_id" in tool_config:
+            self.tool_id = tool_config["tool_id"]
+        elif user_id:
+            # Fallback for backward compatibility or testing
+            self.tool_id = f"default_{user_id}"
+        else:
+            # Last resort fallback (shouldn't happen in normal use)
+            self.tool_id = str(uuid.uuid4())
+
+        db = MongoDB.get_client()[settings.MONGO_DB_NAME]
+        self.collection = db["memories"]
+
+    # -----------------------------
+    # Action implementations
+    # -----------------------------
+    def execute_action(self, action_name: str, **kwargs: Any) -> str:
+        """Execute an action by name.
+
+        Args:
+            action_name: One of view, create, str_replace, insert, delete, rename.
+            **kwargs: Parameters for the action.
+
+        Returns:
+            A human-readable string result.
+        """
+        if not self.user_id:
+            return "Error: MemoryTool requires a valid user_id."
+
+        if action_name == "view":
+            return self._view(
+                kwargs.get("path", "/"),
+                kwargs.get("view_range")
+            )
+
+        if action_name == "create":
+            return self._create(
+                kwargs.get("path", ""),
+                kwargs.get("file_text", "")
+            )
+
+        if action_name == "str_replace":
+            return self._str_replace(
+                kwargs.get("path", ""),
+                kwargs.get("old_str", ""),
+                kwargs.get("new_str", "")
+            )
+
+        if action_name == "insert":
+            return self._insert(
+                kwargs.get("path", ""),
+                kwargs.get("insert_line", 1),
+                kwargs.get("insert_text", "")
+            )
+
+        if action_name == "delete":
+            return self._delete(kwargs.get("path", ""))
+
+        if action_name == "rename":
+            return self._rename(
+                kwargs.get("old_path", ""),
+                kwargs.get("new_path", "")
+            )
+
+        return f"Unknown action: {action_name}"
+
+    def get_actions_metadata(self) -> List[Dict[str, Any]]:
+        """Return JSON metadata describing supported actions for tool schemas."""
+        return [
+            {
+                "name": "view",
+                "description": "Shows directory contents or file contents with optional line ranges.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "path": {
+                            "type": "string",
+                            "description": "Path to file or directory (e.g., /notes.txt or /project/ or /)."
+                        },
+                        "view_range": {
+                            "type": "array",
+                            "items": {"type": "integer"},
+                            "description": "Optional [start_line, end_line] to view specific lines (1-indexed)."
+                        }
+                    },
+                    "required": ["path"]
+                },
+            },
+            {
+                "name": "create",
+                "description": "Create or overwrite a file.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "path": {
+                            "type": "string",
+                            "description": "File path to create (e.g., /notes.txt or /project/task.txt)."
+                        },
+                        "file_text": {
+                            "type": "string",
+                            "description": "Content to write to the file."
+                        }
+                    },
+                    "required": ["path", "file_text"]
+                },
+            },
+            {
+                "name": "str_replace",
+                "description": "Replace text in a file.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "path": {
+                            "type": "string",
+                            "description": "File path (e.g., /notes.txt)."
+                        },
+                        "old_str": {
+                            "type": "string",
+                            "description": "String to find."
+                        },
+                        "new_str": {
+                            "type": "string",
+                            "description": "String to replace with."
+                        }
+                    },
+                    "required": ["path", "old_str", "new_str"]
+                },
+            },
+            {
+                "name": "insert",
+                "description": "Insert text at a specific line in a file.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "path": {
+                            "type": "string",
+                            "description": "File path (e.g., /notes.txt)."
+                        },
+                        "insert_line": {
+                            "type": "integer",
+                            "description": "Line number to insert at (1-indexed)."
+                        },
+                        "insert_text": {
+                            "type": "string",
+                            "description": "Text to insert."
+                        }
+                    },
+                    "required": ["path", "insert_line", "insert_text"]
+                },
+            },
+            {
+                "name": "delete",
+                "description": "Delete a file or directory.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "path": {
+                            "type": "string",
+                            "description": "Path to delete (e.g., /notes.txt or /project/)."
+                        }
+                    },
+                    "required": ["path"]
+                },
+            },
+            {
+                "name": "rename",
+                "description": "Rename or move a file/directory.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "old_path": {
+                            "type": "string",
+                            "description": "Current path (e.g., /old.txt)."
+                        },
+                        "new_path": {
+                            "type": "string",
+                            "description": "New path (e.g., /new.txt)."
+                        }
+                    },
+                    "required": ["old_path", "new_path"]
+                },
+            },
+        ]
+
+    def get_config_requirements(self) -> Dict[str, Any]:
+        """Return configuration requirements."""
+        return {}
+
+    # -----------------------------
+    # Path validation
+    # -----------------------------
+    def _validate_path(self, path: str) -> Optional[str]:
+        """Validate and normalize path.
+
+        Args:
+            path: User-provided path.
+
+        Returns:
+            Normalized path or None if invalid.
+        """
+        if not path:
+            return None
+
+        # Remove any leading/trailing whitespace
+        path = path.strip()
+
+        # Preserve whether path ends with / (indicates directory)
+        is_directory = path.endswith("/")
+
+        # Ensure path starts with / for consistency
+        if not path.startswith("/"):
+            path = "/" + path
+
+        # Check for directory traversal patterns
+        if ".." in path or path.count("//") > 0:
+            return None
+
+        # Normalize the path
+        try:
+            # Convert to Path object and resolve to canonical form
+            normalized = str(Path(path).as_posix())
+
+            # Ensure it still starts with /
+            if not normalized.startswith("/"):
+                return None
+
+            # Preserve trailing slash for directories
+            if is_directory and not normalized.endswith("/") and normalized != "/":
+                normalized = normalized + "/"
+
+            return normalized
+        except Exception:
+            return None
+
+    # -----------------------------
+    # Internal helpers
+    # -----------------------------
+    def _view(self, path: str, view_range: Optional[List[int]] = None) -> str:
+        """View directory contents or file contents."""
+        validated_path = self._validate_path(path)
+        if not validated_path:
+            return "Error: Invalid path."
+
+        # Check if viewing directory (ends with / or is root)
+        if validated_path == "/" or validated_path.endswith("/"):
+            return self._view_directory(validated_path)
+
+        # Otherwise view file
+        return self._view_file(validated_path, view_range)
+
+    def _view_directory(self, path: str) -> str:
+        """List files in a directory."""
+        # Ensure path ends with / for proper prefix matching
+        search_path = path if path.endswith("/") else path + "/"
+
+        # Find all files that start with this directory path
+        query = {
+            "user_id": self.user_id,
+            "tool_id": self.tool_id,
+            "path": {"$regex": f"^{re.escape(search_path)}"}
+        }
+
+        docs = list(self.collection.find(query, {"path": 1}))
+
+        if not docs:
+            return f"Directory: {path}\n(empty)"
+
+        # Extract filenames relative to the directory
+        files = []
+        for doc in docs:
+            file_path = doc["path"]
+            # Remove the directory prefix
+            if file_path.startswith(search_path):
+                relative = file_path[len(search_path):]
+                if relative:
+                    files.append(relative)
+
+        files.sort()
+        file_list = "\n".join(f"- {f}" for f in files)
+        return f"Directory: {path}\n{file_list}"
+
+    def _view_file(self, path: str, view_range: Optional[List[int]] = None) -> str:
+        """View file contents with optional line range."""
+        doc = self.collection.find_one({"user_id": self.user_id, "tool_id": self.tool_id, "path": path})
+
+        if not doc or not doc.get("content"):
+            return f"Error: File not found: {path}"
+
+        content = str(doc["content"])
+
+        # Apply view_range if specified
+        if view_range and len(view_range) == 2:
+            lines = content.split("\n")
+            start, end = view_range
+            # Convert to 0-indexed
+            start_idx = max(0, start - 1)
+            end_idx = min(len(lines), end)
+
+            if start_idx >= len(lines):
+                return f"Error: Line range out of bounds. File has {len(lines)} lines."
+
+            selected_lines = lines[start_idx:end_idx]
+            # Add line numbers (enumerate with 1-based start)
+            numbered_lines = [f"{i}: {line}" for i, line in enumerate(selected_lines, start=start)]
+            return "\n".join(numbered_lines)
+
+        return content
+
+    def _create(self, path: str, file_text: str) -> str:
+        """Create or overwrite a file."""
+        validated_path = self._validate_path(path)
+        if not validated_path:
+            return "Error: Invalid path."
+
+        if validated_path == "/" or validated_path.endswith("/"):
+            return "Error: Cannot create a file at directory path."
+
+        self.collection.update_one(
+            {"user_id": self.user_id, "tool_id": self.tool_id, "path": validated_path},
+            {
+                "$set": {
+                    "content": file_text,
+                    "updated_at": datetime.now()
+                }
+            },
+            upsert=True
+        )
+
+        return f"File created: {validated_path}"
+
+    def _str_replace(self, path: str, old_str: str, new_str: str) -> str:
+        """Replace text in a file."""
+        validated_path = self._validate_path(path)
+        if not validated_path:
+            return "Error: Invalid path."
+
+        if not old_str:
+            return "Error: old_str is required."
+
+        doc = self.collection.find_one({"user_id": self.user_id, "tool_id": self.tool_id, "path": validated_path})
+
+        if not doc or not doc.get("content"):
+            return f"Error: File not found: {validated_path}"
+
+        current_content = str(doc["content"])
+
+        # Check if old_str exists (case-insensitive)
+        if old_str.lower() not in current_content.lower():
+            return f"Error: String '{old_str}' not found in file."
+
+        # Replace the string (case-insensitive)
+        import re as regex_module
+        updated_content = regex_module.sub(regex_module.escape(old_str), new_str, current_content, flags=regex_module.IGNORECASE)
+
+        self.collection.update_one(
+            {"user_id": self.user_id, "tool_id": self.tool_id, "path": validated_path},
+            {
+                "$set": {
+                    "content": updated_content,
+                    "updated_at": datetime.now()
+                }
+            }
+        )
+
+        return f"File updated: {validated_path}"
+
+    def _insert(self, path: str, insert_line: int, insert_text: str) -> str:
+        """Insert text at a specific line."""
+        validated_path = self._validate_path(path)
+        if not validated_path:
+            return "Error: Invalid path."
+
+        if not insert_text:
+            return "Error: insert_text is required."
+
+        doc = self.collection.find_one({"user_id": self.user_id, "tool_id": self.tool_id, "path": validated_path})
+
+        if not doc or not doc.get("content"):
+            return f"Error: File not found: {validated_path}"
+
+        current_content = str(doc["content"])
+        lines = current_content.split("\n")
+
+        # Convert to 0-indexed
+        index = insert_line - 1
+        if index < 0 or index > len(lines):
+            return f"Error: Invalid line number. File has {len(lines)} lines."
+
+        lines.insert(index, insert_text)
+        updated_content = "\n".join(lines)
+
+        self.collection.update_one(
+            {"user_id": self.user_id, "tool_id": self.tool_id, "path": validated_path},
+            {
+                "$set": {
+                    "content": updated_content,
+                    "updated_at": datetime.now()
+                }
+            }
+        )
+
+        return f"Text inserted at line {insert_line} in {validated_path}"
+
+    def _delete(self, path: str) -> str:
+        """Delete a file or directory."""
+        validated_path = self._validate_path(path)
+        if not validated_path:
+            return "Error: Invalid path."
+
+        if validated_path == "/":
+            # Delete all files for this user and tool
+            result = self.collection.delete_many({"user_id": self.user_id, "tool_id": self.tool_id})
+            return f"Deleted {result.deleted_count} file(s) from memory."
+
+        # Check if it's a directory (ends with /)
+        if validated_path.endswith("/"):
+            # Delete all files in directory
+            result = self.collection.delete_many({
+                "user_id": self.user_id,
+                "tool_id": self.tool_id,
+                "path": {"$regex": f"^{re.escape(validated_path)}"}
+            })
+            return f"Deleted directory and {result.deleted_count} file(s)."
+
+        # Try to delete as directory first (without trailing slash)
+        # Check if any files start with this path + /
+        search_path = validated_path + "/"
+        directory_result = self.collection.delete_many({
+            "user_id": self.user_id,
+            "tool_id": self.tool_id,
+            "path": {"$regex": f"^{re.escape(search_path)}"}
+        })
+
+        if directory_result.deleted_count > 0:
+            return f"Deleted directory and {directory_result.deleted_count} file(s)."
+
+        # Delete single file
+        result = self.collection.delete_one({
+            "user_id": self.user_id,
+            "tool_id": self.tool_id,
+            "path": validated_path
+        })
+
+        if result.deleted_count:
+            return f"Deleted: {validated_path}"
+        return f"Error: File not found: {validated_path}"
+
+    def _rename(self, old_path: str, new_path: str) -> str:
+        """Rename or move a file/directory."""
+        validated_old = self._validate_path(old_path)
+        validated_new = self._validate_path(new_path)
+
+        if not validated_old or not validated_new:
+            return "Error: Invalid path."
+
+        if validated_old == "/" or validated_new == "/":
+            return "Error: Cannot rename root directory."
+
+        # Check if renaming a directory
+        if validated_old.endswith("/"):
+            # Ensure validated_new also ends with / for proper path replacement
+            if not validated_new.endswith("/"):
+                validated_new = validated_new + "/"
+
+            # Find all files in the old directory
+            docs = list(self.collection.find({
+                "user_id": self.user_id,
+                "tool_id": self.tool_id,
+                "path": {"$regex": f"^{re.escape(validated_old)}"}
+            }))
+
+            if not docs:
+                return f"Error: Directory not found: {validated_old}"
+
+            # Update paths for all files
+            for doc in docs:
+                old_file_path = doc["path"]
+                new_file_path = old_file_path.replace(validated_old, validated_new, 1)
+
+                self.collection.update_one(
+                    {"_id": doc["_id"]},
+                    {"$set": {"path": new_file_path, "updated_at": datetime.now()}}
+                )
+
+            return f"Renamed directory: {validated_old} -> {validated_new} ({len(docs)} files)"
+
+        # Rename single file
+        doc = self.collection.find_one({
+            "user_id": self.user_id,
+            "tool_id": self.tool_id,
+            "path": validated_old
+        })
+
+        if not doc:
+            return f"Error: File not found: {validated_old}"
+
+        # Check if new path already exists
+        existing = self.collection.find_one({
+            "user_id": self.user_id,
+            "tool_id": self.tool_id,
+            "path": validated_new
+        })
+
+        if existing:
+            return f"Error: File already exists at {validated_new}"
+
+        # Delete the old document and create a new one with the new path
+        self.collection.delete_one({"user_id": self.user_id, "tool_id": self.tool_id, "path": validated_old})
+        self.collection.insert_one({
+            "user_id": self.user_id,
+            "tool_id": self.tool_id,
+            "path": validated_new,
+            "content": doc.get("content", ""),
+            "updated_at": datetime.now()
+        })
+
+        return f"Renamed: {validated_old} -> {validated_new}"

application/agents/tools/notes.py

@@ -0,0 +1,199 @@
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+import uuid
+
+from .base import Tool
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+
+
+class NotesTool(Tool):
+    """Notepad
+
+    Single note. Supports viewing, overwriting, string replacement.
+    """
+
+    def __init__(self, tool_config: Optional[Dict[str, Any]] = None, user_id: Optional[str] = None) -> None:
+        """Initialize the tool.
+
+        Args:
+            tool_config: Optional tool configuration. Should include:
+                - tool_id: Unique identifier for this notes tool instance (from user_tools._id)
+                           This ensures each user's tool configuration has isolated notes
+            user_id: The authenticated user's id (should come from decoded_token["sub"]).
+        """
+        self.user_id: Optional[str] = user_id
+
+        # Get tool_id from configuration (passed from user_tools._id in production)
+        # In production, tool_id is the MongoDB ObjectId string from user_tools collection
+        if tool_config and "tool_id" in tool_config:
+            self.tool_id = tool_config["tool_id"]
+        elif user_id:
+            # Fallback for backward compatibility or testing
+            self.tool_id = f"default_{user_id}"
+        else:
+            # Last resort fallback (shouldn't happen in normal use)
+            self.tool_id = str(uuid.uuid4())
+
+        db = MongoDB.get_client()[settings.MONGO_DB_NAME]
+        self.collection = db["notes"]
+
+    # -----------------------------
+    # Action implementations
+    # -----------------------------
+    def execute_action(self, action_name: str, **kwargs: Any) -> str:
+        """Execute an action by name.
+
+        Args:
+            action_name: One of view, overwrite, str_replace, insert, delete.
+            **kwargs: Parameters for the action.
+
+        Returns:
+            A human-readable string result.
+        """
+        if not self.user_id:
+             return "Error: NotesTool requires a valid user_id."
+
+        if action_name == "view":
+            return self._get_note()
+
+        if action_name == "overwrite":
+            return self._overwrite_note(kwargs.get("text", ""))
+
+        if action_name == "str_replace":
+            return self._str_replace(kwargs.get("old_str", ""), kwargs.get("new_str", ""))
+
+        if action_name == "insert":
+            return self._insert(kwargs.get("line_number", 1), kwargs.get("text", ""))
+
+        if action_name == "delete":
+            return self._delete_note()
+
+        return f"Unknown action: {action_name}"
+
+    def get_actions_metadata(self) -> List[Dict[str, Any]]:
+        """Return JSON metadata describing supported actions for tool schemas."""
+        return [
+            {
+                "name": "view",
+                "description": "Retrieve the user's note.",
+                "parameters": {"type": "object", "properties": {}},
+            },
+            {
+                "name": "overwrite",
+                "description": "Replace the entire note content (creates if doesn't exist).",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "text": {"type": "string", "description": "New note content."}
+                    },
+                    "required": ["text"],
+                },
+            },
+            {
+                "name": "str_replace",
+                "description": "Replace occurrences of old_str with new_str in the note.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "old_str": {"type": "string", "description": "String to find."},
+                        "new_str": {"type": "string", "description": "String to replace with."}
+                    },
+                    "required": ["old_str", "new_str"],
+                },
+            },
+            {
+                "name": "insert",
+                "description": "Insert text at the specified line number (1-indexed).",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "line_number": {"type": "integer", "description": "Line number to insert at (1-indexed)."},
+                        "text": {"type": "string", "description": "Text to insert."}
+                    },
+                    "required": ["line_number", "text"],
+                },
+            },
+            {
+                "name": "delete",
+                "description": "Delete the user's note.",
+                "parameters": {"type": "object", "properties": {}},
+            },
+        ]
+
+    def get_config_requirements(self) -> Dict[str, Any]:
+        """Return configuration requirements (none for now)."""
+        return {}
+
+    # -----------------------------
+    # Internal helpers (single-note)
+    # -----------------------------
+    def _get_note(self) -> str:
+        doc = self.collection.find_one({"user_id": self.user_id, "tool_id": self.tool_id})
+        if not doc or not doc.get("note"):
+            return "No note found."
+        return str(doc["note"])
+
+    def _overwrite_note(self, content: str) -> str:
+        content = (content or "").strip()
+        if not content:
+            return "Note content required."
+        self.collection.update_one(
+            {"user_id": self.user_id, "tool_id": self.tool_id},
+            {"$set": {"note": content, "updated_at": datetime.utcnow()}},
+            upsert=True,  # ✅ create if missing
+        )
+        return "Note saved."
+
+    def _str_replace(self, old_str: str, new_str: str) -> str:
+        if not old_str:
+            return "old_str is required."
+
+        doc = self.collection.find_one({"user_id": self.user_id, "tool_id": self.tool_id})
+        if not doc or not doc.get("note"):
+            return "No note found."
+
+        current_note = str(doc["note"])
+
+        # Case-insensitive search
+        if old_str.lower() not in current_note.lower():
+            return f"String '{old_str}' not found in note."
+
+        # Case-insensitive replacement
+        import re
+        updated_note = re.sub(re.escape(old_str), new_str, current_note, flags=re.IGNORECASE)
+
+        self.collection.update_one(
+            {"user_id": self.user_id, "tool_id": self.tool_id},
+            {"$set": {"note": updated_note, "updated_at": datetime.utcnow()}},
+        )
+        return "Note updated."
+
+    def _insert(self, line_number: int, text: str) -> str:
+        if not text:
+            return "Text is required."
+
+        doc = self.collection.find_one({"user_id": self.user_id, "tool_id": self.tool_id})
+        if not doc or not doc.get("note"):
+            return "No note found."
+
+        current_note = str(doc["note"])
+        lines = current_note.split("\n")
+
+        # Convert to 0-indexed and validate
+        index = line_number - 1
+        if index < 0 or index > len(lines):
+            return f"Invalid line number. Note has {len(lines)} lines."
+
+        lines.insert(index, text)
+        updated_note = "\n".join(lines)
+
+        self.collection.update_one(
+            {"user_id": self.user_id, "tool_id": self.tool_id},
+            {"$set": {"note": updated_note, "updated_at": datetime.utcnow()}},
+        )
+        return "Text inserted."
+
+    def _delete_note(self) -> str:
+        res = self.collection.delete_one({"user_id": self.user_id, "tool_id": self.tool_id})
+        return "Note deleted." if res.deleted_count else "No note found to delete."

application/agents/tools/ntfy.py

@@ -0,0 +1,127 @@
+import requests
+from application.agents.tools.base import Tool
+
+class NtfyTool(Tool):
+    """
+    Ntfy Tool
+    A tool for sending notifications to ntfy topics on a specified server.
+    """
+
+    def __init__(self, config):
+        """
+        Initialize the NtfyTool with configuration.
+
+        Args:
+            config (dict): Configuration dictionary containing the access token.
+        """
+        self.config = config
+        self.token = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        """
+        Execute the specified action with given parameters.
+
+        Args:
+            action_name (str): Name of the action to execute.
+            **kwargs: Parameters for the action, including server_url.
+
+        Returns:
+            dict: Result of the action with status code and message.
+
+        Raises:
+            ValueError: If the action name is unknown.
+        """
+        actions = {
+            "ntfy_send_message": self._send_message,
+        }
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _send_message(self, server_url, message, topic, title=None, priority=None):
+        """
+        Send a message to an ntfy topic on the specified server.
+
+        Args:
+            server_url (str): Base URL of the ntfy server (e.g., https://ntfy.sh).
+            message (str): The message text to send.
+            topic (str): The topic to send the message to.
+            title (str, optional): Title of the notification.
+            priority (int, optional): Priority of the notification (1-5).
+
+        Returns:
+            dict: Response with status code and a confirmation message.
+
+        Raises:
+            ValueError: If priority is not an integer between 1 and 5.
+        """
+        url = f"{server_url.rstrip('/')}/{topic}"
+        headers = {}
+        if title:
+            headers["X-Title"] = title
+        if priority:
+            try:
+                priority = int(priority)
+            except (ValueError, TypeError):
+                raise ValueError("Priority must be convertible to an integer")
+            if priority < 1 or priority > 5:
+                raise ValueError("Priority must be an integer between 1 and 5")
+            headers["X-Priority"] = str(priority)
+        if self.token:
+            headers["Authorization"] = f"Basic {self.token}"
+        data = message.encode("utf-8")
+        response = requests.post(url, headers=headers, data=data)
+        return {"status_code": response.status_code, "message": "Message sent"}
+
+    def get_actions_metadata(self):
+        """
+        Provide metadata about available actions.
+
+        Returns:
+            list: List of dictionaries describing each action.
+        """
+        return [
+            {
+                "name": "ntfy_send_message",
+                "description": "Send a notification to an ntfy topic",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "server_url": {
+                            "type": "string",
+                            "description": "Base URL of the ntfy server",
+                        },
+                        "message": {
+                            "type": "string",
+                            "description": "Text to send in the notification",
+                        },
+                        "topic": {
+                            "type": "string",
+                            "description": "Topic to send the notification to",
+                        },
+                        "title": {
+                            "type": "string",
+                            "description": "Title of the notification (optional)",
+                        },
+                        "priority": {
+                            "type": "integer",
+                            "description": "Priority of the notification (1-5, optional)",
+                        },
+                    },
+                    "required": ["server_url", "message", "topic"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        """
+        Specify the configuration requirements.
+
+        Returns:
+            dict: Dictionary describing required config parameters.
+        """
+        return {
+            "token": {"type": "string", "description": "Access token for authentication"},
+        }

application/agents/tools/postgres.py

@@ -0,0 +1,163 @@
+import psycopg2
+from application.agents.tools.base import Tool
+
+class PostgresTool(Tool):
+    """
+    PostgreSQL Database Tool
+    A tool for connecting to a PostgreSQL database using a connection string,
+    executing SQL queries, and retrieving schema information.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.connection_string = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "postgres_execute_sql": self._execute_sql,
+            "postgres_get_schema": self._get_schema,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _execute_sql(self, sql_query):
+        """
+        Executes an SQL query against the PostgreSQL database using a connection string.
+        """
+        conn = None  # Initialize conn to None for error handling
+        try:
+            conn = psycopg2.connect(self.connection_string)
+            cur = conn.cursor()
+            cur.execute(sql_query)
+            conn.commit()
+
+            if sql_query.strip().lower().startswith("select"):
+                column_names = [desc[0] for desc in cur.description] if cur.description else []
+                results = []
+                rows = cur.fetchall()
+                for row in rows:
+                    results.append(dict(zip(column_names, row)))
+                response_data = {"data": results, "column_names": column_names}
+            else:
+                row_count = cur.rowcount
+                response_data = {"message": f"Query executed successfully, {row_count} rows affected."}
+
+            cur.close()
+            return {
+                "status_code": 200,
+                "message": "SQL query executed successfully.",
+                "response_data": response_data,
+            }
+
+        except psycopg2.Error as e:
+            error_message = f"Database error: {e}"
+            print(f"Database error: {e}")
+            return {
+                "status_code": 500,
+                "message": "Failed to execute SQL query.",
+                "error": error_message,
+            }
+        finally:
+            if conn:  # Ensure connection is closed even if errors occur
+                conn.close()
+
+    def _get_schema(self, db_name):
+        """
+        Retrieves the schema of the PostgreSQL database using a connection string.
+        """
+        conn = None # Initialize conn to None for error handling
+        try:
+            conn = psycopg2.connect(self.connection_string)
+            cur = conn.cursor()
+
+            cur.execute("""
+                SELECT
+                    table_name,
+                    column_name,
+                    data_type,
+                    column_default,
+                    is_nullable
+                FROM
+                    information_schema.columns
+                WHERE
+                    table_schema = 'public'
+                ORDER BY
+                    table_name,
+                    ordinal_position;
+            """)
+
+            schema_data = {}
+            for row in cur.fetchall():
+                table_name, column_name, data_type, column_default, is_nullable = row
+                if table_name not in schema_data:
+                    schema_data[table_name] = []
+                schema_data[table_name].append({
+                    "column_name": column_name,
+                    "data_type": data_type,
+                    "column_default": column_default,
+                    "is_nullable": is_nullable
+                })
+
+            cur.close()
+            return {
+                "status_code": 200,
+                "message": "Database schema retrieved successfully.",
+                "schema": schema_data,
+            }
+
+        except psycopg2.Error as e:
+            error_message = f"Database error: {e}"
+            print(f"Database error: {e}")
+            return {
+                "status_code": 500,
+                "message": "Failed to retrieve database schema.",
+                "error": error_message,
+            }
+        finally:
+            if conn: # Ensure connection is closed even if errors occur
+                conn.close()
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "postgres_execute_sql",
+                "description": "Execute an SQL query against the PostgreSQL database and return the results. Use this tool to interact with the database, e.g., retrieve specific data or perform updates. Only SELECT queries will return data, other queries will return execution status.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "sql_query": {
+                            "type": "string",
+                            "description": "The SQL query to execute.",
+                        },
+                    },
+                    "required": ["sql_query"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "postgres_get_schema",
+                "description": "Retrieve the schema of the PostgreSQL database, including tables and their columns. Use this to understand the database structure before executing queries. db_name is 'default' if not provided.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "db_name": {
+                            "type": "string",
+                            "description": "The name of the database to retrieve the schema for.",
+                        },
+                    },
+                    "required": ["db_name"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {
+                "type": "string",
+                "description": "PostgreSQL database connection string (e.g., 'postgresql://user:password@host:port/dbname')",
+            },
+        }

application/agents/tools/read_webpage.py

@@ -0,0 +1,83 @@
+import requests
+from markdownify import markdownify
+from application.agents.tools.base import Tool
+from urllib.parse import urlparse
+
+class ReadWebpageTool(Tool):
+    """
+    Read Webpage (browser)
+    A tool to fetch the HTML content of a URL and convert it to Markdown.
+    """
+
+    def __init__(self, config=None):
+        """
+        Initializes the tool.
+        :param config: Optional configuration dictionary. Not used by this tool.
+        """
+        self.config = config
+
+    def execute_action(self, action_name: str, **kwargs) -> str:
+        """
+        Executes the specified action. For this tool, the only action is 'read_webpage'.
+
+        :param action_name: The name of the action to execute. Should be 'read_webpage'.
+        :param kwargs: Keyword arguments, must include 'url'.
+        :return: The Markdown content of the webpage or an error message.
+        """
+        if action_name != "read_webpage":
+            return f"Error: Unknown action '{action_name}'. This tool only supports 'read_webpage'."
+
+        url = kwargs.get("url")
+        if not url:
+            return "Error: URL parameter is missing."
+
+        # Ensure the URL has a scheme (if not, default to http)
+        parsed_url = urlparse(url)
+        if not parsed_url.scheme:
+            url = "http://" + url
+        
+        try:
+            response = requests.get(url, timeout=10, headers={'User-Agent': 'DocsGPT-Agent/1.0'})
+            response.raise_for_status()  # Raise an exception for HTTP errors (4xx or 5xx)
+            
+            html_content = response.text
+            #soup = BeautifulSoup(html_content, 'html.parser')
+            
+            
+            markdown_content = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
+            
+            return markdown_content
+
+        except requests.exceptions.RequestException as e:
+            return f"Error fetching URL {url}: {e}"
+        except Exception as e:
+            return f"Error processing URL {url}: {e}"
+
+    def get_actions_metadata(self):
+        """
+        Returns metadata for the actions supported by this tool.
+        """
+        return [
+            {
+                "name": "read_webpage",
+                "description": "Fetches the HTML content of a given URL and returns it as clean Markdown text. Input must be a valid URL.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "url": {
+                            "type": "string",
+                            "description": "The fully qualified URL of the webpage to read (e.g., 'https://www.example.com').",
+                        }
+                    },
+                    "required": ["url"],
+                    "additionalProperties": False,
+                },
+            }
+        ]
+
+    def get_config_requirements(self):
+        """
+        Returns a dictionary describing the configuration requirements for the tool.
+        This tool does not require any specific configuration.
+        """
+        return {}

application/agents/tools/telegram.py

@@ -0,0 +1,86 @@
+import requests
+from application.agents.tools.base import Tool
+
+
+class TelegramTool(Tool):
+    """
+    Telegram Bot
+    A flexible Telegram tool for performing various actions (e.g., sending messages, images).
+    Requires a bot token and chat ID for configuration
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.token = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "telegram_send_message": self._send_message,
+            "telegram_send_image": self._send_image,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _send_message(self, text, chat_id):
+        print(f"Sending message: {text}")
+        url = f"https://api.telegram.org/bot{self.token}/sendMessage"
+        payload = {"chat_id": chat_id, "text": text}
+        response = requests.post(url, data=payload)
+        return {"status_code": response.status_code, "message": "Message sent"}
+
+    def _send_image(self, image_url, chat_id):
+        print(f"Sending image: {image_url}")
+        url = f"https://api.telegram.org/bot{self.token}/sendPhoto"
+        payload = {"chat_id": chat_id, "photo": image_url}
+        response = requests.post(url, data=payload)
+        return {"status_code": response.status_code, "message": "Image sent"}
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "telegram_send_message",
+                "description": "Send a notification to Telegram chat",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "text": {
+                            "type": "string",
+                            "description": "Text to send in the notification",
+                        },
+                        "chat_id": {
+                            "type": "string",
+                            "description": "Chat ID to send the notification to",
+                        },
+                    },
+                    "required": ["text"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "telegram_send_image",
+                "description": "Send an image to the Telegram chat",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "image_url": {
+                            "type": "string",
+                            "description": "URL of the image to send",
+                        },
+                        "chat_id": {
+                            "type": "string",
+                            "description": "Chat ID to send the image to",
+                        },
+                    },
+                    "required": ["image_url"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {"type": "string", "description": "Bot token for authentication"},
+        }

application/agents/tools/todo_list.py

@@ -0,0 +1,321 @@
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+import uuid
+
+from .base import Tool
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+
+
+class TodoListTool(Tool):
+    """Todo List
+
+    Manages todo items for users. Supports creating, viewing, updating, and deleting todos.
+    """
+
+    def __init__(self, tool_config: Optional[Dict[str, Any]] = None, user_id: Optional[str] = None) -> None:
+        """Initialize the tool.
+
+        Args:
+            tool_config: Optional tool configuration. Should include:
+                - tool_id: Unique identifier for this todo list tool instance (from user_tools._id)
+                           This ensures each user's tool configuration has isolated todos
+            user_id: The authenticated user's id (should come from decoded_token["sub"]).
+        """
+        self.user_id: Optional[str] = user_id
+
+        # Get tool_id from configuration (passed from user_tools._id in production)
+        # In production, tool_id is the MongoDB ObjectId string from user_tools collection
+        if tool_config and "tool_id" in tool_config:
+            self.tool_id = tool_config["tool_id"]
+        elif user_id:
+            # Fallback for backward compatibility or testing
+            self.tool_id = f"default_{user_id}"
+        else:
+            # Last resort fallback (shouldn't happen in normal use)
+            self.tool_id = str(uuid.uuid4())
+
+        db = MongoDB.get_client()[settings.MONGO_DB_NAME]
+        self.collection = db["todos"]
+
+    # -----------------------------
+    # Action implementations
+    # -----------------------------
+    def execute_action(self, action_name: str, **kwargs: Any) -> str:
+        """Execute an action by name.
+
+        Args:
+            action_name: One of list, create, get, update, complete, delete.
+            **kwargs: Parameters for the action.
+
+        Returns:
+            A human-readable string result.
+        """
+        if not self.user_id:
+            return "Error: TodoListTool requires a valid user_id."
+
+        if action_name == "list":
+            return self._list()
+
+        if action_name == "create":
+            return self._create(kwargs.get("title", ""))
+
+        if action_name == "get":
+            return self._get(kwargs.get("todo_id"))
+
+        if action_name == "update":
+            return self._update(
+                kwargs.get("todo_id"),
+                kwargs.get("title", "")
+            )
+
+        if action_name == "complete":
+            return self._complete(kwargs.get("todo_id"))
+
+        if action_name == "delete":
+            return self._delete(kwargs.get("todo_id"))
+
+        return f"Unknown action: {action_name}"
+
+    def get_actions_metadata(self) -> List[Dict[str, Any]]:
+        """Return JSON metadata describing supported actions for tool schemas."""
+        return [
+            {
+                "name": "list",
+                "description": "List all todos for the user.",
+                "parameters": {"type": "object", "properties": {}},
+            },
+            {
+                "name": "create",
+                "description": "Create a new todo item.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "title": {
+                            "type": "string",
+                            "description": "Title of the todo item."
+                        }
+                    },
+                    "required": ["title"],
+                },
+            },
+            {
+                "name": "get",
+                "description": "Get a specific todo by ID.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "todo_id": {
+                            "type": "integer",
+                            "description": "The ID of the todo to retrieve."
+                        }
+                    },
+                    "required": ["todo_id"],
+                },
+            },
+            {
+                "name": "update",
+                "description": "Update a todo's title by ID.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "todo_id": {
+                            "type": "integer",
+                            "description": "The ID of the todo to update."
+                        },
+                        "title": {
+                            "type": "string",
+                            "description": "The new title for the todo."
+                        }
+                    },
+                    "required": ["todo_id", "title"],
+                },
+            },
+            {
+                "name": "complete",
+                "description": "Mark a todo as completed.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "todo_id": {
+                            "type": "integer",
+                            "description": "The ID of the todo to mark as completed."
+                        }
+                    },
+                    "required": ["todo_id"],
+                },
+            },
+            {
+                "name": "delete",
+                "description": "Delete a specific todo by ID.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "todo_id": {
+                            "type": "integer",
+                            "description": "The ID of the todo to delete."
+                        }
+                    },
+                    "required": ["todo_id"],
+                },
+            },
+        ]
+
+    def get_config_requirements(self) -> Dict[str, Any]:
+        """Return configuration requirements."""
+        return {}
+
+    # -----------------------------
+    # Internal helpers
+    # -----------------------------
+    def _coerce_todo_id(self, value: Optional[Any]) -> Optional[int]:
+        """Convert todo identifiers to sequential integers."""
+        if value is None:
+            return None
+
+        if isinstance(value, int):
+            return value if value > 0 else None
+
+        if isinstance(value, str):
+            stripped = value.strip()
+            if stripped.isdigit():
+                numeric_value = int(stripped)
+                return numeric_value if numeric_value > 0 else None
+
+        return None
+
+    def _get_next_todo_id(self) -> int:
+        """Get the next sequential todo_id for this user and tool.
+
+        Returns a simple integer (1, 2, 3, ...) scoped to this user/tool.
+        With 5-10 todos max, scanning is negligible.
+        """
+        # Find all todos for this user/tool and get their IDs
+        todos = list(self.collection.find(
+            {"user_id": self.user_id, "tool_id": self.tool_id},
+            {"todo_id": 1}
+        ))
+
+        # Find the maximum todo_id
+        max_id = 0
+        for todo in todos:
+            todo_id = self._coerce_todo_id(todo.get("todo_id"))
+            if todo_id is not None:
+                max_id = max(max_id, todo_id)
+
+        return max_id + 1
+
+    def _list(self) -> str:
+        """List all todos for the user."""
+        cursor = self.collection.find({"user_id": self.user_id, "tool_id": self.tool_id})
+        todos = list(cursor)
+
+        if not todos:
+            return "No todos found."
+
+        result_lines = ["Todos:"]
+        for doc in todos:
+            todo_id = doc.get("todo_id")
+            title = doc.get("title", "Untitled")
+            status = doc.get("status", "open")
+
+            line = f"[{todo_id}] {title} ({status})"
+            result_lines.append(line)
+
+        return "\n".join(result_lines)
+
+    def _create(self, title: str) -> str:
+        """Create a new todo item."""
+        title = (title or "").strip()
+        if not title:
+            return "Error: Title is required."
+
+        now = datetime.now()
+        todo_id = self._get_next_todo_id()
+
+        doc = {
+            "todo_id": todo_id,
+            "user_id": self.user_id,
+            "tool_id": self.tool_id,
+            "title": title,
+            "status": "open",
+            "created_at": now,
+            "updated_at": now,
+        }
+        self.collection.insert_one(doc)
+        return f"Todo created with ID {todo_id}: {title}"
+
+    def _get(self, todo_id: Optional[Any]) -> str:
+        """Get a specific todo by ID."""
+        parsed_todo_id = self._coerce_todo_id(todo_id)
+        if parsed_todo_id is None:
+            return "Error: todo_id must be a positive integer."
+
+        doc = self.collection.find_one({
+            "user_id": self.user_id,
+            "tool_id": self.tool_id,
+            "todo_id": parsed_todo_id
+        })
+
+        if not doc:
+            return f"Error: Todo with ID {parsed_todo_id} not found."
+
+        title = doc.get("title", "Untitled")
+        status = doc.get("status", "open")
+
+        result = f"Todo [{parsed_todo_id}]:\nTitle: {title}\nStatus: {status}"
+
+        return result
+
+    def _update(self, todo_id: Optional[Any], title: str) -> str:
+        """Update a todo's title by ID."""
+        parsed_todo_id = self._coerce_todo_id(todo_id)
+        if parsed_todo_id is None:
+            return "Error: todo_id must be a positive integer."
+
+        title = (title or "").strip()
+        if not title:
+            return "Error: Title is required."
+
+        result = self.collection.update_one(
+            {"user_id": self.user_id, "tool_id": self.tool_id, "todo_id": parsed_todo_id},
+            {"$set": {"title": title, "updated_at": datetime.now()}}
+        )
+
+        if result.matched_count == 0:
+            return f"Error: Todo with ID {parsed_todo_id} not found."
+
+        return f"Todo {parsed_todo_id} updated to: {title}"
+
+    def _complete(self, todo_id: Optional[Any]) -> str:
+        """Mark a todo as completed."""
+        parsed_todo_id = self._coerce_todo_id(todo_id)
+        if parsed_todo_id is None:
+            return "Error: todo_id must be a positive integer."
+
+        result = self.collection.update_one(
+            {"user_id": self.user_id, "tool_id": self.tool_id, "todo_id": parsed_todo_id},
+            {"$set": {"status": "completed", "updated_at": datetime.now()}}
+        )
+
+        if result.matched_count == 0:
+            return f"Error: Todo with ID {parsed_todo_id} not found."
+
+        return f"Todo {parsed_todo_id} marked as completed."
+
+    def _delete(self, todo_id: Optional[Any]) -> str:
+        """Delete a specific todo by ID."""
+        parsed_todo_id = self._coerce_todo_id(todo_id)
+        if parsed_todo_id is None:
+            return "Error: todo_id must be a positive integer."
+
+        result = self.collection.delete_one({
+            "user_id": self.user_id,
+            "tool_id": self.tool_id,
+            "todo_id": parsed_todo_id
+        })
+
+        if result.deleted_count == 0:
+            return f"Error: Todo with ID {parsed_todo_id} not found."
+
+        return f"Todo {parsed_todo_id} deleted."

application/agents/tools/tool_action_parser.py

@@ -0,0 +1,69 @@
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ToolActionParser:
+    def __init__(self, llm_type):
+        self.llm_type = llm_type
+        self.parsers = {
+            "OpenAILLM": self._parse_openai_llm,
+            "GoogleLLM": self._parse_google_llm,
+        }
+
+    def parse_args(self, call):
+        parser = self.parsers.get(self.llm_type, self._parse_openai_llm)
+        return parser(call)
+
+    def _parse_openai_llm(self, call):
+        try:
+            call_args = json.loads(call.arguments)
+            tool_parts = call.name.split("_")
+
+            # If the tool name doesn't contain an underscore, it's likely a hallucinated tool
+            if len(tool_parts) < 2:
+                logger.warning(
+                    f"Invalid tool name format: {call.name}. Expected format: action_name_tool_id"
+                )
+                return None, None, None
+
+            tool_id = tool_parts[-1]
+            action_name = "_".join(tool_parts[:-1])
+
+            # Validate that tool_id looks like a numerical ID
+            if not tool_id.isdigit():
+                logger.warning(
+                    f"Tool ID '{tool_id}' is not numerical. This might be a hallucinated tool call."
+                )
+
+        except (AttributeError, TypeError, json.JSONDecodeError) as e:
+            logger.error(f"Error parsing OpenAI LLM call: {e}")
+            return None, None, None
+        return tool_id, action_name, call_args
+
+    def _parse_google_llm(self, call):
+        try:
+            call_args = call.arguments
+            tool_parts = call.name.split("_")
+
+            # If the tool name doesn't contain an underscore, it's likely a hallucinated tool
+            if len(tool_parts) < 2:
+                logger.warning(
+                    f"Invalid tool name format: {call.name}. Expected format: action_name_tool_id"
+                )
+                return None, None, None
+
+            tool_id = tool_parts[-1]
+            action_name = "_".join(tool_parts[:-1])
+
+            # Validate that tool_id looks like a numerical ID
+            if not tool_id.isdigit():
+                logger.warning(
+                    f"Tool ID '{tool_id}' is not numerical. This might be a hallucinated tool call."
+                )
+
+        except (AttributeError, TypeError) as e:
+            logger.error(f"Error parsing Google LLM call: {e}")
+            return None, None, None
+        return tool_id, action_name, call_args

application/agents/tools/tool_manager.py

@@ -0,0 +1,49 @@
+import importlib
+import inspect
+import os
+import pkgutil
+
+from application.agents.tools.base import Tool
+
+
+class ToolManager:
+    def __init__(self, config):
+        self.config = config
+        self.tools = {}
+        self.load_tools()
+
+    def load_tools(self):
+        tools_dir = os.path.join(os.path.dirname(__file__))
+        for finder, name, ispkg in pkgutil.iter_modules([tools_dir]):
+            if name == "base" or name.startswith("__"):
+                continue
+            module = importlib.import_module(f"application.agents.tools.{name}")
+            for member_name, obj in inspect.getmembers(module, inspect.isclass):
+                if issubclass(obj, Tool) and obj is not Tool:
+                    tool_config = self.config.get(name, {})
+                    self.tools[name] = obj(tool_config)
+
+    def load_tool(self, tool_name, tool_config, user_id=None):
+        self.config[tool_name] = tool_config
+        module = importlib.import_module(f"application.agents.tools.{tool_name}")
+        for member_name, obj in inspect.getmembers(module, inspect.isclass):
+            if issubclass(obj, Tool) and obj is not Tool:
+                if tool_name in {"mcp_tool", "notes", "memory", "todo_list"} and user_id:
+                    return obj(tool_config, user_id)
+                else:
+                    return obj(tool_config)
+
+    def execute_action(self, tool_name, action_name, user_id=None, **kwargs):
+        if tool_name not in self.tools:
+            raise ValueError(f"Tool '{tool_name}' not loaded")
+        if tool_name in {"mcp_tool", "memory", "todo_list"} and user_id:
+            tool_config = self.config.get(tool_name, {})
+            tool = self.load_tool(tool_name, tool_config, user_id)
+            return tool.execute_action(action_name, **kwargs)
+        return self.tools[tool_name].execute_action(action_name, **kwargs)
+
+    def get_all_actions_metadata(self):
+        metadata = []
+        for tool in self.tools.values():
+            metadata.extend(tool.get_actions_metadata())
+        return metadata

application/api/__init__.py

@@ -0,0 +1,7 @@
+from flask_restx import Api
+
+api = Api(
+    version="1.0",
+    title="DocsGPT API",
+    description="API for DocsGPT",
+)

application/api/answer/__init__.py

@@ -0,0 +1,19 @@
+from flask import Blueprint
+
+from application.api import api
+from application.api.answer.routes.answer import AnswerResource
+from application.api.answer.routes.base import answer_ns
+from application.api.answer.routes.stream import StreamResource
+
+
+answer = Blueprint("answer", __name__)
+
+api.add_namespace(answer_ns)
+
+
+def init_answer_routes():
+    api.add_resource(StreamResource, "/stream")
+    api.add_resource(AnswerResource, "/api/answer")
+
+
+init_answer_routes()

application/api/answer/routes/answer.py

@@ -0,0 +1,142 @@
+import logging
+import traceback
+
+from flask import make_response, request
+from flask_restx import fields, Resource
+
+from application.api import api
+
+from application.api.answer.routes.base import answer_ns, BaseAnswerResource
+
+from application.api.answer.services.stream_processor import StreamProcessor
+
+logger = logging.getLogger(__name__)
+
+
+@answer_ns.route("/api/answer")
+class AnswerResource(Resource, BaseAnswerResource):
+    def __init__(self, *args, **kwargs):
+        Resource.__init__(self, *args, **kwargs)
+        BaseAnswerResource.__init__(self)
+
+    answer_model = answer_ns.model(
+        "AnswerModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
+            ),
+            "history": fields.List(
+                fields.String,
+                required=False,
+                description="Conversation history (only for new conversations)",
+            ),
+            "conversation_id": fields.String(
+                required=False,
+                description="Existing conversation ID (loads history)",
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+            "save_conversation": fields.Boolean(
+                required=False,
+                default=True,
+                description="Whether to save the conversation",
+            ),
+            "model_id": fields.String(
+                required=False,
+                description="Model ID to use for this request",
+            ),
+            "passthrough": fields.Raw(
+                required=False,
+                description="Dynamic parameters to inject into prompt template",
+            ),
+        },
+    )
+
+    @api.expect(answer_model)
+    @api.doc(description="Provide a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        if error := self.validate_request(data):
+            return error
+        decoded_token = getattr(request, "decoded_token", None)
+        processor = StreamProcessor(data, decoded_token)
+        try:
+            processor.initialize()
+            if not processor.decoded_token:
+                return make_response({"error": "Unauthorized"}, 401)
+
+            docs_together, docs_list = processor.pre_fetch_docs(
+                data.get("question", "")
+            )
+            tools_data = processor.pre_fetch_tools()
+
+            agent = processor.create_agent(
+                docs_together=docs_together,
+                docs=docs_list,
+                tools_data=tools_data,
+            )
+
+            if error := self.check_usage(processor.agent_config):
+                return error
+
+            stream = self.complete_stream(
+                question=data["question"],
+                agent=agent,
+                conversation_id=processor.conversation_id,
+                user_api_key=processor.agent_config.get("user_api_key"),
+                decoded_token=processor.decoded_token,
+                isNoneDoc=data.get("isNoneDoc"),
+                index=None,
+                should_save_conversation=data.get("save_conversation", True),
+                model_id=processor.model_id,
+            )
+            stream_result = self.process_response_stream(stream)
+
+            if len(stream_result) == 7:
+                (
+                    conversation_id,
+                    response,
+                    sources,
+                    tool_calls,
+                    thought,
+                    error,
+                    structured_info,
+                ) = stream_result
+            else:
+                conversation_id, response, sources, tool_calls, thought, error = (
+                    stream_result
+                )
+                structured_info = None
+
+            if error:
+                return make_response({"error": error}, 400)
+            result = {
+                "conversation_id": conversation_id,
+                "answer": response,
+                "sources": sources,
+                "tool_calls": tool_calls,
+                "thought": thought,
+            }
+
+            if structured_info:
+                result.update(structured_info)
+        except Exception as e:
+            logger.error(
+                f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return make_response({"error": str(e)}, 500)
+        return make_response(result, 200)

application/api/answer/routes/base.py

@@ -0,0 +1,442 @@
+import datetime
+import json
+import logging
+from typing import Any, Dict, Generator, List, Optional
+
+from flask import jsonify, make_response, Response
+from flask_restx import Namespace
+
+from application.api.answer.services.conversation_service import ConversationService
+from application.core.model_utils import (
+    get_api_key_for_provider,
+    get_default_model_id,
+    get_provider_from_model_id,
+)
+
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.llm.llm_creator import LLMCreator
+from application.utils import check_required_fields
+
+logger = logging.getLogger(__name__)
+
+
+answer_ns = Namespace("answer", description="Answer related operations", path="/")
+
+
+class BaseAnswerResource:
+    """Shared base class for answer endpoints"""
+
+    def __init__(self):
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        self.db = db
+        self.user_logs_collection = db["user_logs"]
+        self.default_model_id = get_default_model_id()
+        self.conversation_service = ConversationService()
+
+    def validate_request(
+        self, data: Dict[str, Any], require_conversation_id: bool = False
+    ) -> Optional[Response]:
+        """Common request validation"""
+        required_fields = ["question"]
+        if require_conversation_id:
+            required_fields.append("conversation_id")
+        if missing_fields := check_required_fields(data, required_fields):
+            return missing_fields
+        return None
+
+    def check_usage(self, agent_config: Dict) -> Optional[Response]:
+        """Check if there is a usage limit and if it is exceeded
+
+        Args:
+            agent_config: The config dict of agent instance
+
+        Returns:
+            None or Response if either of limits exceeded.
+
+        """
+        api_key = agent_config.get("user_api_key")
+        if not api_key:
+            return None
+        agents_collection = self.db["agents"]
+        agent = agents_collection.find_one({"key": api_key})
+
+        if not agent:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid API key."}), 401
+            )
+        limited_token_mode_raw = agent.get("limited_token_mode", False)
+        limited_request_mode_raw = agent.get("limited_request_mode", False)
+
+        limited_token_mode = (
+            limited_token_mode_raw
+            if isinstance(limited_token_mode_raw, bool)
+            else limited_token_mode_raw == "True"
+        )
+        limited_request_mode = (
+            limited_request_mode_raw
+            if isinstance(limited_request_mode_raw, bool)
+            else limited_request_mode_raw == "True"
+        )
+
+        token_limit = int(
+            agent.get("token_limit", settings.DEFAULT_AGENT_LIMITS["token_limit"])
+        )
+        request_limit = int(
+            agent.get("request_limit", settings.DEFAULT_AGENT_LIMITS["request_limit"])
+        )
+
+        token_usage_collection = self.db["token_usage"]
+
+        end_date = datetime.datetime.now()
+        start_date = end_date - datetime.timedelta(hours=24)
+
+        match_query = {
+            "timestamp": {"$gte": start_date, "$lte": end_date},
+            "api_key": api_key,
+        }
+
+        if limited_token_mode:
+            token_pipeline = [
+                {"$match": match_query},
+                {
+                    "$group": {
+                        "_id": None,
+                        "total_tokens": {
+                            "$sum": {"$add": ["$prompt_tokens", "$generated_tokens"]}
+                        },
+                    }
+                },
+            ]
+            token_result = list(token_usage_collection.aggregate(token_pipeline))
+            daily_token_usage = token_result[0]["total_tokens"] if token_result else 0
+        else:
+            daily_token_usage = 0
+        if limited_request_mode:
+            daily_request_usage = token_usage_collection.count_documents(match_query)
+        else:
+            daily_request_usage = 0
+        if not limited_token_mode and not limited_request_mode:
+            return None
+        token_exceeded = (
+            limited_token_mode and token_limit > 0 and daily_token_usage >= token_limit
+        )
+        request_exceeded = (
+            limited_request_mode
+            and request_limit > 0
+            and daily_request_usage >= request_limit
+        )
+
+        if token_exceeded or request_exceeded:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Exceeding usage limit, please try again later.",
+                    }
+                ),
+                429,
+            )
+        return None
+
+    def complete_stream(
+        self,
+        question: str,
+        agent: Any,
+        conversation_id: Optional[str],
+        user_api_key: Optional[str],
+        decoded_token: Dict[str, Any],
+        isNoneDoc: bool = False,
+        index: Optional[int] = None,
+        should_save_conversation: bool = True,
+        attachment_ids: Optional[List[str]] = None,
+        agent_id: Optional[str] = None,
+        is_shared_usage: bool = False,
+        shared_token: Optional[str] = None,
+        model_id: Optional[str] = None,
+    ) -> Generator[str, None, None]:
+        """
+        Generator function that streams the complete conversation response.
+
+        Args:
+            question: The user's question
+            agent: The agent instance
+            retriever: The retriever instance
+            conversation_id: Existing conversation ID
+            user_api_key: User's API key if any
+            decoded_token: Decoded JWT token
+            isNoneDoc: Flag for document-less responses
+            index: Index of message to update
+            should_save_conversation: Whether to persist the conversation
+            attachment_ids: List of attachment IDs
+            agent_id: ID of agent used
+            is_shared_usage: Flag for shared agent usage
+            shared_token: Token for shared agent
+            model_id: Model ID used for the request
+            retrieved_docs: Pre-fetched documents for sources (optional)
+
+        Yields:
+            Server-sent event strings
+        """
+        try:
+            response_full, thought, source_log_docs, tool_calls = "", "", [], []
+            is_structured = False
+            schema_info = None
+            structured_chunks = []
+
+            for line in agent.gen(query=question):
+                if "answer" in line:
+                    response_full += str(line["answer"])
+                    if line.get("structured"):
+                        is_structured = True
+                        schema_info = line.get("schema")
+                        structured_chunks.append(line["answer"])
+                    else:
+                        data = json.dumps({"type": "answer", "answer": line["answer"]})
+                        yield f"data: {data}\n\n"
+                elif "sources" in line:
+                    truncated_sources = []
+                    source_log_docs = line["sources"]
+                    for source in line["sources"]:
+                        truncated_source = source.copy()
+                        if "text" in truncated_source:
+                            truncated_source["text"] = (
+                                truncated_source["text"][:100].strip() + "..."
+                            )
+                        truncated_sources.append(truncated_source)
+                    if truncated_sources:
+                        data = json.dumps(
+                            {"type": "source", "source": truncated_sources}
+                        )
+                        yield f"data: {data}\n\n"
+                elif "tool_calls" in line:
+                    tool_calls = line["tool_calls"]
+                    data = json.dumps({"type": "tool_calls", "tool_calls": tool_calls})
+                    yield f"data: {data}\n\n"
+                elif "thought" in line:
+                    thought += line["thought"]
+                    data = json.dumps({"type": "thought", "thought": line["thought"]})
+                    yield f"data: {data}\n\n"
+                elif "type" in line:
+                    data = json.dumps(line)
+                    yield f"data: {data}\n\n"
+            if is_structured and structured_chunks:
+                structured_data = {
+                    "type": "structured_answer",
+                    "answer": response_full,
+                    "structured": True,
+                    "schema": schema_info,
+                }
+                data = json.dumps(structured_data)
+                yield f"data: {data}\n\n"
+            if isNoneDoc:
+                for doc in source_log_docs:
+                    doc["source"] = "None"
+            provider = (
+                get_provider_from_model_id(model_id)
+                if model_id
+                else settings.LLM_PROVIDER
+            )
+            system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER)
+
+            llm = LLMCreator.create_llm(
+                provider or settings.LLM_PROVIDER,
+                api_key=system_api_key,
+                user_api_key=user_api_key,
+                decoded_token=decoded_token,
+                model_id=model_id,
+            )
+
+            if should_save_conversation:
+                conversation_id = self.conversation_service.save_conversation(
+                    conversation_id,
+                    question,
+                    response_full,
+                    thought,
+                    source_log_docs,
+                    tool_calls,
+                    llm,
+                    model_id or self.default_model_id,
+                    decoded_token,
+                    index=index,
+                    api_key=user_api_key,
+                    agent_id=agent_id,
+                    is_shared_usage=is_shared_usage,
+                    shared_token=shared_token,
+                    attachment_ids=attachment_ids,
+                )
+                # Persist compression metadata/summary if it exists and wasn't saved mid-execution
+                compression_meta = getattr(agent, "compression_metadata", None)
+                compression_saved = getattr(agent, "compression_saved", False)
+                if conversation_id and compression_meta and not compression_saved:
+                    try:
+                        self.conversation_service.update_compression_metadata(
+                            conversation_id, compression_meta
+                        )
+                        self.conversation_service.append_compression_message(
+                            conversation_id, compression_meta
+                        )
+                        agent.compression_saved = True
+                        logger.info(
+                            f"Persisted compression metadata for conversation {conversation_id}"
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Failed to persist compression metadata: {str(e)}",
+                            exc_info=True,
+                        )
+            else:
+                conversation_id = None
+            id_data = {"type": "id", "id": str(conversation_id)}
+            data = json.dumps(id_data)
+            yield f"data: {data}\n\n"
+
+            log_data = {
+                "action": "stream_answer",
+                "level": "info",
+                "user": decoded_token.get("sub"),
+                "api_key": user_api_key,
+                "question": question,
+                "response": response_full,
+                "sources": source_log_docs,
+                "attachments": attachment_ids,
+                "timestamp": datetime.datetime.now(datetime.timezone.utc),
+            }
+            if is_structured:
+                log_data["structured_output"] = True
+                if schema_info:
+                    log_data["schema"] = schema_info
+            # Clean up text fields to be no longer than 10000 characters
+
+            for key, value in log_data.items():
+                if isinstance(value, str) and len(value) > 10000:
+                    log_data[key] = value[:10000]
+            self.user_logs_collection.insert_one(log_data)
+
+            data = json.dumps({"type": "end"})
+            yield f"data: {data}\n\n"
+        except GeneratorExit:
+            logger.info(f"Stream aborted by client for question: {question[:50]}... ")
+            # Save partial response
+
+            if should_save_conversation and response_full:
+                try:
+                    if isNoneDoc:
+                        for doc in source_log_docs:
+                            doc["source"] = "None"
+                    llm = LLMCreator.create_llm(
+                        settings.LLM_PROVIDER,
+                        api_key=settings.API_KEY,
+                        user_api_key=user_api_key,
+                        decoded_token=decoded_token,
+                    )
+                    self.conversation_service.save_conversation(
+                        conversation_id,
+                        question,
+                        response_full,
+                        thought,
+                        source_log_docs,
+                        tool_calls,
+                        llm,
+                        model_id or self.default_model_id,
+                        decoded_token,
+                        index=index,
+                        api_key=user_api_key,
+                        agent_id=agent_id,
+                        is_shared_usage=is_shared_usage,
+                        shared_token=shared_token,
+                        attachment_ids=attachment_ids,
+                    )
+                    compression_meta = getattr(agent, "compression_metadata", None)
+                    compression_saved = getattr(agent, "compression_saved", False)
+                    if conversation_id and compression_meta and not compression_saved:
+                        try:
+                            self.conversation_service.update_compression_metadata(
+                                conversation_id, compression_meta
+                            )
+                            self.conversation_service.append_compression_message(
+                                conversation_id, compression_meta
+                            )
+                            agent.compression_saved = True
+                            logger.info(
+                                f"Persisted compression metadata for conversation {conversation_id} (partial stream)"
+                            )
+                        except Exception as e:
+                            logger.error(
+                                f"Failed to persist compression metadata (partial stream): {str(e)}",
+                                exc_info=True,
+                            )
+                except Exception as e:
+                    logger.error(
+                        f"Error saving partial response: {str(e)}", exc_info=True
+                    )
+            raise
+        except Exception as e:
+            logger.error(f"Error in stream: {str(e)}", exc_info=True)
+            data = json.dumps(
+                {
+                    "type": "error",
+                    "error": "Please try again later. We apologize for any inconvenience.",
+                }
+            )
+            yield f"data: {data}\n\n"
+            return
+
+    def process_response_stream(self, stream):
+        """Process the stream response for non-streaming endpoint"""
+        conversation_id = ""
+        response_full = ""
+        source_log_docs = []
+        tool_calls = []
+        thought = ""
+        stream_ended = False
+        is_structured = False
+        schema_info = None
+
+        for line in stream:
+            try:
+                event_data = line.replace("data: ", "").strip()
+                event = json.loads(event_data)
+
+                if event["type"] == "id":
+                    conversation_id = event["id"]
+                elif event["type"] == "answer":
+                    response_full += event["answer"]
+                elif event["type"] == "structured_answer":
+                    response_full = event["answer"]
+                    is_structured = True
+                    schema_info = event.get("schema")
+                elif event["type"] == "source":
+                    source_log_docs = event["source"]
+                elif event["type"] == "tool_calls":
+                    tool_calls = event["tool_calls"]
+                elif event["type"] == "thought":
+                    thought = event["thought"]
+                elif event["type"] == "error":
+                    logger.error(f"Error from stream: {event['error']}")
+                    return None, None, None, None, event["error"], None
+                elif event["type"] == "end":
+                    stream_ended = True
+            except (json.JSONDecodeError, KeyError) as e:
+                logger.warning(f"Error parsing stream event: {e}, line: {line}")
+                continue
+        if not stream_ended:
+            logger.error("Stream ended unexpectedly without an 'end' event.")
+            return None, None, None, None, "Stream ended unexpectedly", None
+        result = (
+            conversation_id,
+            response_full,
+            source_log_docs,
+            tool_calls,
+            thought,
+            None,
+        )
+
+        if is_structured:
+            result = result + ({"structured": True, "schema": schema_info},)
+        return result
+
+    def error_stream_generate(self, err_response):
+        data = json.dumps({"type": "error", "error": err_response})
+        yield f"data: {data}\n\n"

application/api/answer/routes/stream.py

@@ -0,0 +1,132 @@
+import logging
+import traceback
+
+from flask import request, Response
+from flask_restx import fields, Resource
+
+from application.api import api
+
+from application.api.answer.routes.base import answer_ns, BaseAnswerResource
+
+from application.api.answer.services.stream_processor import StreamProcessor
+
+logger = logging.getLogger(__name__)
+
+
+@answer_ns.route("/stream")
+class StreamResource(Resource, BaseAnswerResource):
+    def __init__(self, *args, **kwargs):
+        Resource.__init__(self, *args, **kwargs)
+        BaseAnswerResource.__init__(self)
+
+    stream_model = answer_ns.model(
+        "StreamModel",
+        {
+            "question": fields.String(
+                required=True, description="Question to be asked"
+            ),
+            "history": fields.List(
+                fields.String,
+                required=False,
+                description="Conversation history (only for new conversations)",
+            ),
+            "conversation_id": fields.String(
+                required=False,
+                description="Existing conversation ID (loads history)",
+            ),
+            "prompt_id": fields.String(
+                required=False, default="default", description="Prompt ID"
+            ),
+            "chunks": fields.Integer(
+                required=False, default=2, description="Number of chunks"
+            ),
+            "token_limit": fields.Integer(required=False, description="Token limit"),
+            "retriever": fields.String(required=False, description="Retriever type"),
+            "api_key": fields.String(required=False, description="API key"),
+            "active_docs": fields.String(
+                required=False, description="Active documents"
+            ),
+            "isNoneDoc": fields.Boolean(
+                required=False, description="Flag indicating if no document is used"
+            ),
+            "index": fields.Integer(
+                required=False, description="Index of the query to update"
+            ),
+            "save_conversation": fields.Boolean(
+                required=False,
+                default=True,
+                description="Whether to save the conversation",
+            ),
+            "model_id": fields.String(
+                required=False,
+                description="Model ID to use for this request",
+            ),
+            "attachments": fields.List(
+                fields.String, required=False, description="List of attachment IDs"
+            ),
+            "passthrough": fields.Raw(
+                required=False,
+                description="Dynamic parameters to inject into prompt template",
+            ),
+        },
+    )
+
+    @api.expect(stream_model)
+    @api.doc(description="Stream a response based on the question and retriever")
+    def post(self):
+        data = request.get_json()
+        if error := self.validate_request(data, "index" in data):
+            return error
+        decoded_token = getattr(request, "decoded_token", None)
+        processor = StreamProcessor(data, decoded_token)
+        try:
+            processor.initialize()
+
+            docs_together, docs_list = processor.pre_fetch_docs(data["question"])
+            tools_data = processor.pre_fetch_tools()
+
+            agent = processor.create_agent(
+                docs_together=docs_together, docs=docs_list, tools_data=tools_data
+            )
+
+            if error := self.check_usage(processor.agent_config):
+                return error
+            return Response(
+                self.complete_stream(
+                    question=data["question"],
+                    agent=agent,
+                    conversation_id=processor.conversation_id,
+                    user_api_key=processor.agent_config.get("user_api_key"),
+                    decoded_token=processor.decoded_token,
+                    isNoneDoc=data.get("isNoneDoc"),
+                    index=data.get("index"),
+                    should_save_conversation=data.get("save_conversation", True),
+                    attachment_ids=data.get("attachments", []),
+                    agent_id=data.get("agent_id"),
+                    is_shared_usage=processor.is_shared_usage,
+                    shared_token=processor.shared_token,
+                    model_id=processor.model_id,
+                ),
+                mimetype="text/event-stream",
+            )
+        except ValueError as e:
+            message = "Malformed request body"
+            logger.error(
+                f"/stream - error: {message} - specific error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return Response(
+                self.error_stream_generate(message),
+                status=400,
+                mimetype="text/event-stream",
+            )
+        except Exception as e:
+            logger.error(
+                f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
+                extra={"error": str(e), "traceback": traceback.format_exc()},
+            )
+            return Response(
+                self.error_stream_generate("Unknown error occurred"),
+                status=400,
+                mimetype="text/event-stream",
+            )

application/api/answer/services/compression/__init__.py

@@ -0,0 +1,20 @@
+"""
+Compression module for managing conversation context compression.
+
+"""
+
+from application.api.answer.services.compression.orchestrator import (
+    CompressionOrchestrator,
+)
+from application.api.answer.services.compression.service import CompressionService
+from application.api.answer.services.compression.types import (
+    CompressionResult,
+    CompressionMetadata,
+)
+
+__all__ = [
+    "CompressionOrchestrator",
+    "CompressionService",
+    "CompressionResult",
+    "CompressionMetadata",
+]

application/api/answer/services/compression/message_builder.py

@@ -0,0 +1,234 @@
+"""Message reconstruction utilities for compression."""
+
+import logging
+import uuid
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class MessageBuilder:
+    """Builds message arrays from compressed context."""
+
+    @staticmethod
+    def build_from_compressed_context(
+        system_prompt: str,
+        compressed_summary: Optional[str],
+        recent_queries: List[Dict],
+        include_tool_calls: bool = False,
+        context_type: str = "pre_request",
+    ) -> List[Dict]:
+        """
+        Build messages from compressed context.
+
+        Args:
+            system_prompt: Original system prompt
+            compressed_summary: Compressed summary (if any)
+            recent_queries: Recent uncompressed queries
+            include_tool_calls: Whether to include tool calls from history
+            context_type: Type of context ('pre_request' or 'mid_execution')
+
+        Returns:
+            List of message dicts ready for LLM
+        """
+        # Append compression summary to system prompt if present
+        if compressed_summary:
+            system_prompt = MessageBuilder._append_compression_context(
+                system_prompt, compressed_summary, context_type
+            )
+
+        messages = [{"role": "system", "content": system_prompt}]
+
+        # Add recent history
+        for query in recent_queries:
+            if "prompt" in query and "response" in query:
+                messages.append({"role": "user", "content": query["prompt"]})
+                messages.append({"role": "assistant", "content": query["response"]})
+
+            # Add tool calls from history if present
+            if include_tool_calls and "tool_calls" in query:
+                for tool_call in query["tool_calls"]:
+                    call_id = tool_call.get("call_id") or str(uuid.uuid4())
+
+                    function_call_dict = {
+                        "function_call": {
+                            "name": tool_call.get("action_name"),
+                            "args": tool_call.get("arguments"),
+                            "call_id": call_id,
+                        }
+                    }
+                    function_response_dict = {
+                        "function_response": {
+                            "name": tool_call.get("action_name"),
+                            "response": {"result": tool_call.get("result")},
+                            "call_id": call_id,
+                        }
+                    }
+
+                    messages.append(
+                        {"role": "assistant", "content": [function_call_dict]}
+                    )
+                    messages.append(
+                        {"role": "tool", "content": [function_response_dict]}
+                    )
+
+        # If no recent queries (everything was compressed), add a continuation user message
+        if len(recent_queries) == 0 and compressed_summary:
+            messages.append({
+                "role": "user",
+                "content": "Please continue with the remaining tasks based on the context above."
+            })
+            logger.info("Added continuation user message to maintain proper turn-taking after full compression")
+
+        return messages
+
+    @staticmethod
+    def _append_compression_context(
+        system_prompt: str, compressed_summary: str, context_type: str = "pre_request"
+    ) -> str:
+        """
+        Append compression context to system prompt.
+
+        Args:
+            system_prompt: Original system prompt
+            compressed_summary: Summary to append
+            context_type: Type of compression context
+
+        Returns:
+            Updated system prompt
+        """
+        # Remove existing compression context if present
+        if "This session is being continued" in system_prompt or "Context window limit reached" in system_prompt:
+            parts = system_prompt.split("\n\n---\n\n")
+            system_prompt = parts[0]
+
+        # Build appropriate context message based on type
+        if context_type == "mid_execution":
+            context_message = (
+                "\n\n---\n\n"
+                "Context window limit reached during execution. "
+                "Previous conversation has been compressed to fit within limits. "
+                "The conversation is summarized below:\n\n"
+                f"{compressed_summary}"
+            )
+        else:  # pre_request
+            context_message = (
+                "\n\n---\n\n"
+                "This session is being continued from a previous conversation that "
+                "has been compressed to fit within context limits. "
+                "The conversation is summarized below:\n\n"
+                f"{compressed_summary}"
+            )
+
+        return system_prompt + context_message
+
+    @staticmethod
+    def rebuild_messages_after_compression(
+        messages: List[Dict],
+        compressed_summary: Optional[str],
+        recent_queries: List[Dict],
+        include_current_execution: bool = False,
+        include_tool_calls: bool = False,
+    ) -> Optional[List[Dict]]:
+        """
+        Rebuild the message list after compression so tool execution can continue.
+
+        Args:
+            messages: Original message list
+            compressed_summary: Compressed summary
+            recent_queries: Recent uncompressed queries
+            include_current_execution: Whether to preserve current execution messages
+            include_tool_calls: Whether to include tool calls from history
+
+        Returns:
+            Rebuilt message list or None if failed
+        """
+        # Find the system message
+        system_message = next(
+            (msg for msg in messages if msg.get("role") == "system"), None
+        )
+        if not system_message:
+            logger.warning("No system message found in messages list")
+            return None
+
+        # Update system message with compressed summary
+        if compressed_summary:
+            content = system_message.get("content", "")
+            system_message["content"] = MessageBuilder._append_compression_context(
+                content, compressed_summary, "mid_execution"
+            )
+            logger.info(
+                "Appended compression summary to system prompt (truncated): %s",
+                (
+                    compressed_summary[:500] + "..."
+                    if len(compressed_summary) > 500
+                    else compressed_summary
+                ),
+            )
+
+        rebuilt_messages = [system_message]
+
+        # Add recent history from compressed context
+        for query in recent_queries:
+            if "prompt" in query and "response" in query:
+                rebuilt_messages.append({"role": "user", "content": query["prompt"]})
+                rebuilt_messages.append(
+                    {"role": "assistant", "content": query["response"]}
+                )
+
+            # Add tool calls from history if present
+            if include_tool_calls and "tool_calls" in query:
+                for tool_call in query["tool_calls"]:
+                    call_id = tool_call.get("call_id") or str(uuid.uuid4())
+
+                    function_call_dict = {
+                        "function_call": {
+                            "name": tool_call.get("action_name"),
+                            "args": tool_call.get("arguments"),
+                            "call_id": call_id,
+                        }
+                    }
+                    function_response_dict = {
+                        "function_response": {
+                            "name": tool_call.get("action_name"),
+                            "response": {"result": tool_call.get("result")},
+                            "call_id": call_id,
+                        }
+                    }
+
+                    rebuilt_messages.append(
+                        {"role": "assistant", "content": [function_call_dict]}
+                    )
+                    rebuilt_messages.append(
+                        {"role": "tool", "content": [function_response_dict]}
+                    )
+
+        # If no recent queries (everything was compressed), add a continuation user message
+        if len(recent_queries) == 0 and compressed_summary:
+            rebuilt_messages.append({
+                "role": "user",
+                "content": "Please continue with the remaining tasks based on the context above."
+            })
+            logger.info("Added continuation user message to maintain proper turn-taking after full compression")
+
+        if include_current_execution:
+            # Preserve any messages that were added during the current execution cycle
+            recent_msg_count = 1  # system message
+            for query in recent_queries:
+                if "prompt" in query and "response" in query:
+                    recent_msg_count += 2
+                if "tool_calls" in query:
+                    recent_msg_count += len(query["tool_calls"]) * 2
+
+            if len(messages) > recent_msg_count:
+                current_execution_messages = messages[recent_msg_count:]
+                rebuilt_messages.extend(current_execution_messages)
+                logger.info(
+                    f"Preserved {len(current_execution_messages)} messages from current execution cycle"
+                )
+
+        logger.info(
+            f"Messages rebuilt: {len(messages)} → {len(rebuilt_messages)} messages. "
+            f"Ready to continue tool execution."
+        )
+        return rebuilt_messages

application/api/answer/services/compression/orchestrator.py

@@ -0,0 +1,232 @@
+"""High-level compression orchestration."""
+
+import logging
+from typing import Any, Dict, Optional
+
+from application.api.answer.services.compression.service import CompressionService
+from application.api.answer.services.compression.threshold_checker import (
+    CompressionThresholdChecker,
+)
+from application.api.answer.services.compression.types import CompressionResult
+from application.api.answer.services.conversation_service import ConversationService
+from application.core.model_utils import (
+    get_api_key_for_provider,
+    get_provider_from_model_id,
+)
+from application.core.settings import settings
+from application.llm.llm_creator import LLMCreator
+
+logger = logging.getLogger(__name__)
+
+
+class CompressionOrchestrator:
+    """
+    Facade for compression operations.
+
+    Coordinates between all compression components and provides
+    a simple interface for callers.
+    """
+
+    def __init__(
+        self,
+        conversation_service: ConversationService,
+        threshold_checker: Optional[CompressionThresholdChecker] = None,
+    ):
+        """
+        Initialize orchestrator.
+
+        Args:
+            conversation_service: Service for DB operations
+            threshold_checker: Custom threshold checker (optional)
+        """
+        self.conversation_service = conversation_service
+        self.threshold_checker = threshold_checker or CompressionThresholdChecker()
+
+    def compress_if_needed(
+        self,
+        conversation_id: str,
+        user_id: str,
+        model_id: str,
+        decoded_token: Dict[str, Any],
+        current_query_tokens: int = 500,
+    ) -> CompressionResult:
+        """
+        Check if compression is needed and perform it if so.
+
+        This is the main entry point for compression operations.
+
+        Args:
+            conversation_id: Conversation ID
+            user_id: User ID
+            model_id: Model being used for conversation
+            decoded_token: User's decoded JWT token
+            current_query_tokens: Estimated tokens for current query
+
+        Returns:
+            CompressionResult with summary and recent queries
+        """
+        try:
+            # Load conversation
+            conversation = self.conversation_service.get_conversation(
+                conversation_id, user_id
+            )
+
+            if not conversation:
+                logger.warning(
+                    f"Conversation {conversation_id} not found for user {user_id}"
+                )
+                return CompressionResult.failure("Conversation not found")
+
+            # Check if compression is needed
+            if not self.threshold_checker.should_compress(
+                conversation, model_id, current_query_tokens
+            ):
+                # No compression needed, return full history
+                queries = conversation.get("queries", [])
+                return CompressionResult.success_no_compression(queries)
+
+            # Perform compression
+            return self._perform_compression(
+                conversation_id, conversation, model_id, decoded_token
+            )
+
+        except Exception as e:
+            logger.error(
+                f"Error in compress_if_needed: {str(e)}", exc_info=True
+            )
+            return CompressionResult.failure(str(e))
+
+    def _perform_compression(
+        self,
+        conversation_id: str,
+        conversation: Dict[str, Any],
+        model_id: str,
+        decoded_token: Dict[str, Any],
+    ) -> CompressionResult:
+        """
+        Perform the actual compression operation.
+
+        Args:
+            conversation_id: Conversation ID
+            conversation: Conversation document
+            model_id: Model ID for conversation
+            decoded_token: User token
+
+        Returns:
+            CompressionResult
+        """
+        try:
+            # Determine which model to use for compression
+            compression_model = (
+                settings.COMPRESSION_MODEL_OVERRIDE
+                if settings.COMPRESSION_MODEL_OVERRIDE
+                else model_id
+            )
+
+            # Get provider and API key for compression model
+            provider = get_provider_from_model_id(compression_model)
+            api_key = get_api_key_for_provider(provider)
+
+            # Create compression LLM
+            compression_llm = LLMCreator.create_llm(
+                provider,
+                api_key=api_key,
+                user_api_key=None,
+                decoded_token=decoded_token,
+                model_id=compression_model,
+            )
+
+            # Create compression service with DB update capability
+            compression_service = CompressionService(
+                llm=compression_llm,
+                model_id=compression_model,
+                conversation_service=self.conversation_service,
+            )
+
+            # Compress all queries up to the latest
+            queries_count = len(conversation.get("queries", []))
+            compress_up_to = queries_count - 1
+
+            if compress_up_to < 0:
+                logger.warning("No queries to compress")
+                return CompressionResult.success_no_compression([])
+
+            logger.info(
+                f"Initiating compression for conversation {conversation_id}: "
+                f"compressing all {queries_count} queries (0-{compress_up_to})"
+            )
+
+            # Perform compression and save to DB
+            metadata = compression_service.compress_and_save(
+                conversation_id, conversation, compress_up_to
+            )
+
+            logger.info(
+                f"Compression successful - ratio: {metadata.compression_ratio:.1f}x, "
+                f"saved {metadata.original_token_count - metadata.compressed_token_count} tokens"
+            )
+
+            # Reload conversation with updated metadata
+            conversation = self.conversation_service.get_conversation(
+                conversation_id, user_id=decoded_token.get("sub")
+            )
+
+            # Get compressed context
+            compressed_summary, recent_queries = (
+                compression_service.get_compressed_context(conversation)
+            )
+
+            return CompressionResult.success_with_compression(
+                compressed_summary, recent_queries, metadata
+            )
+
+        except Exception as e:
+            logger.error(f"Error performing compression: {str(e)}", exc_info=True)
+            return CompressionResult.failure(str(e))
+
+    def compress_mid_execution(
+        self,
+        conversation_id: str,
+        user_id: str,
+        model_id: str,
+        decoded_token: Dict[str, Any],
+        current_conversation: Optional[Dict[str, Any]] = None,
+    ) -> CompressionResult:
+        """
+        Perform compression during tool execution.
+
+        Args:
+            conversation_id: Conversation ID
+            user_id: User ID
+            model_id: Model ID
+            decoded_token: User token
+            current_conversation: Pre-loaded conversation (optional)
+
+        Returns:
+            CompressionResult
+        """
+        try:
+            # Load conversation if not provided
+            if current_conversation:
+                conversation = current_conversation
+            else:
+                conversation = self.conversation_service.get_conversation(
+                    conversation_id, user_id
+                )
+
+            if not conversation:
+                logger.warning(
+                    f"Could not load conversation {conversation_id} for mid-execution compression"
+                )
+                return CompressionResult.failure("Conversation not found")
+
+            # Perform compression
+            return self._perform_compression(
+                conversation_id, conversation, model_id, decoded_token
+            )
+
+        except Exception as e:
+            logger.error(
+                f"Error in mid-execution compression: {str(e)}", exc_info=True
+            )
+            return CompressionResult.failure(str(e))

application/api/answer/services/compression/prompt_builder.py

@@ -0,0 +1,149 @@
+"""Compression prompt building logic."""
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class CompressionPromptBuilder:
+    """Builds prompts for LLM compression calls."""
+
+    def __init__(self, version: str = "v1.0"):
+        """
+        Initialize prompt builder.
+
+        Args:
+            version: Prompt template version to use
+        """
+        self.version = version
+        self.system_prompt = self._load_prompt(version)
+
+    def _load_prompt(self, version: str) -> str:
+        """
+        Load prompt template from file.
+
+        Args:
+            version: Version string (e.g., 'v1.0')
+
+        Returns:
+            Prompt template content
+
+        Raises:
+            FileNotFoundError: If prompt template file doesn't exist
+        """
+        current_dir = Path(__file__).resolve().parents[4]
+        prompt_path = current_dir / "prompts" / "compression" / f"{version}.txt"
+
+        try:
+            with open(prompt_path, "r") as f:
+                return f.read()
+        except FileNotFoundError:
+            logger.error(f"Compression prompt template not found: {prompt_path}")
+            raise FileNotFoundError(
+                f"Compression prompt template '{version}' not found at {prompt_path}. "
+                f"Please ensure the template file exists."
+            )
+
+    def build_prompt(
+        self,
+        queries: List[Dict[str, Any]],
+        existing_compressions: Optional[List[Dict[str, Any]]] = None,
+    ) -> List[Dict[str, str]]:
+        """
+        Build messages for compression LLM call.
+
+        Args:
+            queries: List of query objects to compress
+            existing_compressions: List of previous compression points
+
+        Returns:
+            List of message dicts for LLM
+        """
+        # Build conversation text
+        conversation_text = self._format_conversation(queries)
+
+        # Add existing compression context if present
+        existing_compression_context = ""
+        if existing_compressions and len(existing_compressions) > 0:
+            existing_compression_context = (
+                "\n\nIMPORTANT: This conversation has been compressed before. "
+                "Previous compression summaries:\n\n"
+            )
+            for i, comp in enumerate(existing_compressions):
+                existing_compression_context += (
+                    f"--- Compression {i + 1} (up to message {comp.get('query_index', 'unknown')}) ---\n"
+                    f"{comp.get('compressed_summary', '')}\n\n"
+                )
+            existing_compression_context += (
+                "Your task is to create a NEW summary that incorporates the context from "
+                "previous compressions AND the new messages below. The final summary should "
+                "be comprehensive and include all important information from both previous "
+                "compressions and new messages.\n\n"
+            )
+
+        user_prompt = (
+            f"{existing_compression_context}"
+            f"Here is the conversation to summarize:\n\n"
+            f"{conversation_text}"
+        )
+
+        messages = [
+            {"role": "system", "content": self.system_prompt},
+            {"role": "user", "content": user_prompt},
+        ]
+
+        return messages
+
+    def _format_conversation(self, queries: List[Dict[str, Any]]) -> str:
+        """
+        Format conversation queries into readable text for compression.
+
+        Args:
+            queries: List of query objects
+
+        Returns:
+            Formatted conversation text
+        """
+        conversation_lines = []
+
+        for i, query in enumerate(queries):
+            conversation_lines.append(f"--- Message {i + 1} ---")
+            conversation_lines.append(f"User: {query.get('prompt', '')}")
+
+            # Add tool calls if present
+            tool_calls = query.get("tool_calls", [])
+            if tool_calls:
+                conversation_lines.append("\nTool Calls:")
+                for tc in tool_calls:
+                    tool_name = tc.get("tool_name", "unknown")
+                    action_name = tc.get("action_name", "unknown")
+                    arguments = tc.get("arguments", {})
+                    result = tc.get("result", "")
+                    if result is None:
+                        result = ""
+                    status = tc.get("status", "unknown")
+
+                    # Include full tool result for complete compression context
+                    conversation_lines.append(
+                        f"  - {tool_name}.{action_name}({arguments}) "
+                        f"[{status}] → {result}"
+                    )
+
+            # Add agent thought if present
+            thought = query.get("thought", "")
+            if thought:
+                conversation_lines.append(f"\nAgent Thought: {thought}")
+
+            # Add assistant response
+            conversation_lines.append(f"\nAssistant: {query.get('response', '')}")
+
+            # Add sources if present
+            sources = query.get("sources", [])
+            if sources:
+                conversation_lines.append(f"\nSources Used: {len(sources)} documents")
+
+            conversation_lines.append("")  # Empty line between messages
+
+        return "\n".join(conversation_lines)

application/api/answer/services/compression/service.py

@@ -0,0 +1,306 @@
+"""Core compression service with simplified responsibilities."""
+
+import logging
+import re
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from application.api.answer.services.compression.prompt_builder import (
+    CompressionPromptBuilder,
+)
+from application.api.answer.services.compression.token_counter import TokenCounter
+from application.api.answer.services.compression.types import (
+    CompressionMetadata,
+)
+from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+class CompressionService:
+    """
+    Service for compressing conversation history.
+
+    Handles DB updates.
+    """
+
+    def __init__(
+        self,
+        llm,
+        model_id: str,
+        conversation_service=None,
+        prompt_builder: Optional[CompressionPromptBuilder] = None,
+    ):
+        """
+        Initialize compression service.
+
+        Args:
+            llm: LLM instance to use for compression
+            model_id: Model ID for compression
+            conversation_service: Service for DB operations (optional, for DB updates)
+            prompt_builder: Custom prompt builder (optional)
+        """
+        self.llm = llm
+        self.model_id = model_id
+        self.conversation_service = conversation_service
+        self.prompt_builder = prompt_builder or CompressionPromptBuilder(
+            version=settings.COMPRESSION_PROMPT_VERSION
+        )
+
+    def compress_conversation(
+        self,
+        conversation: Dict[str, Any],
+        compress_up_to_index: int,
+    ) -> CompressionMetadata:
+        """
+        Compress conversation history up to specified index.
+
+        Args:
+            conversation: Full conversation document
+            compress_up_to_index: Last query index to include in compression
+
+        Returns:
+            CompressionMetadata with compression details
+
+        Raises:
+            ValueError: If compress_up_to_index is invalid
+        """
+        try:
+            queries = conversation.get("queries", [])
+
+            if compress_up_to_index < 0 or compress_up_to_index >= len(queries):
+                raise ValueError(
+                    f"Invalid compress_up_to_index: {compress_up_to_index} "
+                    f"(conversation has {len(queries)} queries)"
+                )
+
+            # Get queries to compress
+            queries_to_compress = queries[: compress_up_to_index + 1]
+
+            # Check if there are existing compressions
+            existing_compressions = conversation.get("compression_metadata", {}).get(
+                "compression_points", []
+            )
+
+            if existing_compressions:
+                logger.info(
+                    f"Found {len(existing_compressions)} previous compression(s) - "
+                    f"will incorporate into new summary"
+                )
+
+            # Calculate original token count
+            original_tokens = TokenCounter.count_query_tokens(queries_to_compress)
+
+            # Log tool call stats
+            self._log_tool_call_stats(queries_to_compress)
+
+            # Build compression prompt
+            messages = self.prompt_builder.build_prompt(
+                queries_to_compress, existing_compressions
+            )
+
+            # Call LLM to generate compression
+            logger.info(
+                f"Starting compression: {len(queries_to_compress)} queries "
+                f"(messages 0-{compress_up_to_index}, {original_tokens} tokens) "
+                f"using model {self.model_id}"
+            )
+
+            response = self.llm.gen(
+                model=self.model_id, messages=messages, max_tokens=4000
+            )
+
+            # Extract summary from response
+            compressed_summary = self._extract_summary(response)
+
+            # Calculate compressed token count
+            compressed_tokens = TokenCounter.count_message_tokens(
+                [{"content": compressed_summary}]
+            )
+
+            # Calculate compression ratio
+            compression_ratio = (
+                original_tokens / compressed_tokens if compressed_tokens > 0 else 0
+            )
+
+            logger.info(
+                f"Compression complete: {original_tokens} → {compressed_tokens} tokens "
+                f"({compression_ratio:.1f}x compression)"
+            )
+
+            # Build compression metadata
+            compression_metadata = CompressionMetadata(
+                timestamp=datetime.now(timezone.utc),
+                query_index=compress_up_to_index,
+                compressed_summary=compressed_summary,
+                original_token_count=original_tokens,
+                compressed_token_count=compressed_tokens,
+                compression_ratio=compression_ratio,
+                model_used=self.model_id,
+                compression_prompt_version=self.prompt_builder.version,
+            )
+
+            return compression_metadata
+
+        except Exception as e:
+            logger.error(f"Error compressing conversation: {str(e)}", exc_info=True)
+            raise
+
+    def compress_and_save(
+        self,
+        conversation_id: str,
+        conversation: Dict[str, Any],
+        compress_up_to_index: int,
+    ) -> CompressionMetadata:
+        """
+        Compress conversation and save to database.
+
+        Args:
+            conversation_id: Conversation ID
+            conversation: Full conversation document
+            compress_up_to_index: Last query index to include
+
+        Returns:
+            CompressionMetadata
+
+        Raises:
+            ValueError: If conversation_service not provided or invalid index
+        """
+        if not self.conversation_service:
+            raise ValueError(
+                "conversation_service required for compress_and_save operation"
+            )
+
+        # Perform compression
+        metadata = self.compress_conversation(conversation, compress_up_to_index)
+
+        # Save to database
+        self.conversation_service.update_compression_metadata(
+            conversation_id, metadata.to_dict()
+        )
+
+        logger.info(f"Compression metadata saved to database for {conversation_id}")
+
+        return metadata
+
+    def get_compressed_context(
+        self, conversation: Dict[str, Any]
+    ) -> tuple[Optional[str], List[Dict[str, Any]]]:
+        """
+        Get compressed summary + recent uncompressed messages.
+
+        Args:
+            conversation: Full conversation document
+
+        Returns:
+            (compressed_summary, recent_messages)
+        """
+        try:
+            compression_metadata = conversation.get("compression_metadata", {})
+
+            if not compression_metadata.get("is_compressed"):
+                logger.debug("No compression metadata found - using full history")
+                queries = conversation.get("queries", [])
+                if queries is None:
+                    logger.error("Conversation queries is None - returning empty list")
+                    return None, []
+                return None, queries
+
+            compression_points = compression_metadata.get("compression_points", [])
+
+            if not compression_points:
+                logger.debug("No compression points found - using full history")
+                queries = conversation.get("queries", [])
+                if queries is None:
+                    logger.error("Conversation queries is None - returning empty list")
+                    return None, []
+                return None, queries
+
+            # Get the most recent compression point
+            latest_compression = compression_points[-1]
+            compressed_summary = latest_compression.get("compressed_summary")
+            last_compressed_index = latest_compression.get("query_index")
+            compressed_tokens = latest_compression.get("compressed_token_count", 0)
+            original_tokens = latest_compression.get("original_token_count", 0)
+
+            # Get only messages after compression point
+            queries = conversation.get("queries", [])
+            total_queries = len(queries)
+            recent_queries = queries[last_compressed_index + 1 :]
+
+            logger.info(
+                f"Using compressed context: summary ({compressed_tokens} tokens, "
+                f"compressed from {original_tokens}) + {len(recent_queries)} recent messages "
+                f"(messages {last_compressed_index + 1}-{total_queries - 1})"
+            )
+
+            return compressed_summary, recent_queries
+
+        except Exception as e:
+            logger.error(
+                f"Error getting compressed context: {str(e)}", exc_info=True
+            )
+            queries = conversation.get("queries", [])
+            if queries is None:
+                return None, []
+            return None, queries
+
+    def _extract_summary(self, llm_response: str) -> str:
+        """
+        Extract clean summary from LLM response.
+
+        Args:
+            llm_response: Raw LLM response
+
+        Returns:
+            Cleaned summary text
+        """
+        try:
+            # Try to extract content within <summary> tags
+            summary_match = re.search(
+                r"<summary>(.*?)</summary>", llm_response, re.DOTALL
+            )
+
+            if summary_match:
+                summary = summary_match.group(1).strip()
+            else:
+                # If no summary tags, remove analysis tags and use the rest
+                summary = re.sub(
+                    r"<analysis>.*?</analysis>", "", llm_response, flags=re.DOTALL
+                ).strip()
+
+            return summary
+
+        except Exception as e:
+            logger.warning(f"Error extracting summary: {str(e)}, using full response")
+            return llm_response
+
+    def _log_tool_call_stats(self, queries: List[Dict[str, Any]]) -> None:
+        """Log statistics about tool calls in queries."""
+        total_tool_calls = 0
+        total_tool_result_chars = 0
+        tool_call_breakdown = {}
+
+        for q in queries:
+            for tc in q.get("tool_calls", []):
+                total_tool_calls += 1
+                tool_name = tc.get("tool_name", "unknown")
+                action_name = tc.get("action_name", "unknown")
+                key = f"{tool_name}.{action_name}"
+                tool_call_breakdown[key] = tool_call_breakdown.get(key, 0) + 1
+
+                # Track total tool result size
+                result = tc.get("result", "")
+                if result:
+                    total_tool_result_chars += len(str(result))
+
+        if total_tool_calls > 0:
+            tool_breakdown_str = ", ".join(
+                f"{tool}({count})"
+                for tool, count in sorted(tool_call_breakdown.items())
+            )
+            tool_result_kb = total_tool_result_chars / 1024
+            logger.info(
+                f"Tool call breakdown: {tool_breakdown_str} "
+                f"(total result size: {tool_result_kb:.1f} KB, {total_tool_result_chars:,} chars)"
+            )

application/api/answer/services/compression/threshold_checker.py

@@ -0,0 +1,103 @@
+"""Compression threshold checking logic."""
+
+import logging
+from typing import Any, Dict
+
+from application.core.model_utils import get_token_limit
+from application.core.settings import settings
+from application.api.answer.services.compression.token_counter import TokenCounter
+
+logger = logging.getLogger(__name__)
+
+
+class CompressionThresholdChecker:
+    """Determines if compression is needed based on token thresholds."""
+
+    def __init__(self, threshold_percentage: float = None):
+        """
+        Initialize threshold checker.
+
+        Args:
+            threshold_percentage: Percentage of context to use as threshold
+                                 (defaults to settings.COMPRESSION_THRESHOLD_PERCENTAGE)
+        """
+        self.threshold_percentage = (
+            threshold_percentage or settings.COMPRESSION_THRESHOLD_PERCENTAGE
+        )
+
+    def should_compress(
+        self,
+        conversation: Dict[str, Any],
+        model_id: str,
+        current_query_tokens: int = 500,
+    ) -> bool:
+        """
+        Determine if compression is needed.
+
+        Args:
+            conversation: Full conversation document
+            model_id: Target model for this request
+            current_query_tokens: Estimated tokens for current query
+
+        Returns:
+            True if tokens >= threshold% of context window
+        """
+        try:
+            # Calculate total tokens in conversation
+            total_tokens = TokenCounter.count_conversation_tokens(conversation)
+            total_tokens += current_query_tokens
+
+            # Get context window limit for model
+            context_limit = get_token_limit(model_id)
+
+            # Calculate threshold
+            threshold = int(context_limit * self.threshold_percentage)
+
+            compression_needed = total_tokens >= threshold
+            percentage_used = (total_tokens / context_limit) * 100
+
+            if compression_needed:
+                logger.warning(
+                    f"COMPRESSION TRIGGERED: {total_tokens} tokens / {context_limit} limit "
+                    f"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%)"
+                )
+            else:
+                logger.info(
+                    f"Compression check: {total_tokens}/{context_limit} tokens "
+                    f"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%) - No compression needed"
+                )
+
+            return compression_needed
+
+        except Exception as e:
+            logger.error(f"Error checking compression need: {str(e)}", exc_info=True)
+            return False
+
+    def check_message_tokens(self, messages: list, model_id: str) -> bool:
+        """
+        Check if message list exceeds threshold.
+
+        Args:
+            messages: List of message dicts
+            model_id: Target model
+
+        Returns:
+            True if at or above threshold
+        """
+        try:
+            current_tokens = TokenCounter.count_message_tokens(messages)
+            context_limit = get_token_limit(model_id)
+            threshold = int(context_limit * self.threshold_percentage)
+
+            if current_tokens >= threshold:
+                logger.warning(
+                    f"Message context limit approaching: {current_tokens}/{context_limit} tokens "
+                    f"({(current_tokens/context_limit)*100:.1f}%)"
+                )
+                return True
+
+            return False
+
+        except Exception as e:
+            logger.error(f"Error checking message tokens: {str(e)}", exc_info=True)
+            return False

application/api/answer/services/compression/token_counter.py

@@ -0,0 +1,103 @@
+"""Token counting utilities for compression."""
+
+import logging
+from typing import Any, Dict, List
+
+from application.utils import num_tokens_from_string
+from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+class TokenCounter:
+    """Centralized token counting for conversations and messages."""
+
+    @staticmethod
+    def count_message_tokens(messages: List[Dict]) -> int:
+        """
+        Calculate total tokens in a list of messages.
+
+        Args:
+            messages: List of message dicts with 'content' field
+
+        Returns:
+            Total token count
+        """
+        total_tokens = 0
+        for message in messages:
+            content = message.get("content", "")
+            if isinstance(content, str):
+                total_tokens += num_tokens_from_string(content)
+            elif isinstance(content, list):
+                # Handle structured content (tool calls, etc.)
+                for item in content:
+                    if isinstance(item, dict):
+                        total_tokens += num_tokens_from_string(str(item))
+        return total_tokens
+
+    @staticmethod
+    def count_query_tokens(
+        queries: List[Dict[str, Any]], include_tool_calls: bool = True
+    ) -> int:
+        """
+        Count tokens across multiple query objects.
+
+        Args:
+            queries: List of query objects from conversation
+            include_tool_calls: Whether to count tool call tokens
+
+        Returns:
+            Total token count
+        """
+        total_tokens = 0
+
+        for query in queries:
+            # Count prompt and response tokens
+            if "prompt" in query:
+                total_tokens += num_tokens_from_string(query["prompt"])
+            if "response" in query:
+                total_tokens += num_tokens_from_string(query["response"])
+            if "thought" in query:
+                total_tokens += num_tokens_from_string(query.get("thought", ""))
+
+            # Count tool call tokens
+            if include_tool_calls and "tool_calls" in query:
+                for tool_call in query["tool_calls"]:
+                    tool_call_string = (
+                        f"Tool: {tool_call.get('tool_name')} | "
+                        f"Action: {tool_call.get('action_name')} | "
+                        f"Args: {tool_call.get('arguments')} | "
+                        f"Response: {tool_call.get('result')}"
+                    )
+                    total_tokens += num_tokens_from_string(tool_call_string)
+
+        return total_tokens
+
+    @staticmethod
+    def count_conversation_tokens(
+        conversation: Dict[str, Any], include_system_prompt: bool = False
+    ) -> int:
+        """
+        Calculate total tokens in a conversation.
+
+        Args:
+            conversation: Conversation document
+            include_system_prompt: Whether to include system prompt in count
+
+        Returns:
+            Total token count
+        """
+        try:
+            queries = conversation.get("queries", [])
+            total_tokens = TokenCounter.count_query_tokens(queries)
+
+            # Add system prompt tokens if requested
+            if include_system_prompt:
+                # Rough estimate for system prompt
+                total_tokens += settings.RESERVED_TOKENS.get("system_prompt", 500)
+
+            return total_tokens
+
+        except Exception as e:
+            logger.error(f"Error calculating conversation tokens: {str(e)}")
+            return 0

application/api/answer/services/compression/types.py

@@ -0,0 +1,83 @@
+"""Type definitions for compression module."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class CompressionMetadata:
+    """Metadata about a compression operation."""
+
+    timestamp: datetime
+    query_index: int
+    compressed_summary: str
+    original_token_count: int
+    compressed_token_count: int
+    compression_ratio: float
+    model_used: str
+    compression_prompt_version: str
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for DB storage."""
+        return {
+            "timestamp": self.timestamp,
+            "query_index": self.query_index,
+            "compressed_summary": self.compressed_summary,
+            "original_token_count": self.original_token_count,
+            "compressed_token_count": self.compressed_token_count,
+            "compression_ratio": self.compression_ratio,
+            "model_used": self.model_used,
+            "compression_prompt_version": self.compression_prompt_version,
+        }
+
+
+@dataclass
+class CompressionResult:
+    """Result of a compression operation."""
+
+    success: bool
+    compressed_summary: Optional[str] = None
+    recent_queries: List[Dict[str, Any]] = field(default_factory=list)
+    metadata: Optional[CompressionMetadata] = None
+    error: Optional[str] = None
+    compression_performed: bool = False
+
+    @classmethod
+    def success_with_compression(
+        cls, summary: str, queries: List[Dict], metadata: CompressionMetadata
+    ) -> "CompressionResult":
+        """Create a successful result with compression."""
+        return cls(
+            success=True,
+            compressed_summary=summary,
+            recent_queries=queries,
+            metadata=metadata,
+            compression_performed=True,
+        )
+
+    @classmethod
+    def success_no_compression(cls, queries: List[Dict]) -> "CompressionResult":
+        """Create a successful result without compression needed."""
+        return cls(
+            success=True,
+            recent_queries=queries,
+            compression_performed=False,
+        )
+
+    @classmethod
+    def failure(cls, error: str) -> "CompressionResult":
+        """Create a failure result."""
+        return cls(success=False, error=error, compression_performed=False)
+
+    def as_history(self) -> List[Dict[str, str]]:
+        """
+        Convert recent queries to history format.
+
+        Returns:
+            List of prompt/response dicts
+        """
+        return [
+            {"prompt": q["prompt"], "response": q["response"]}
+            for q in self.recent_queries
+        ]

application/api/answer/services/conversation_service.py

@@ -0,0 +1,282 @@
+import logging
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from application.core.mongo_db import MongoDB
+
+from application.core.settings import settings
+from bson import ObjectId
+
+
+logger = logging.getLogger(__name__)
+
+
+class ConversationService:
+    def __init__(self):
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        self.conversations_collection = db["conversations"]
+        self.agents_collection = db["agents"]
+
+    def get_conversation(
+        self, conversation_id: str, user_id: str
+    ) -> Optional[Dict[str, Any]]:
+        """Retrieve a conversation with proper access control"""
+        if not conversation_id or not user_id:
+            return None
+        try:
+            conversation = self.conversations_collection.find_one(
+                {
+                    "_id": ObjectId(conversation_id),
+                    "$or": [{"user": user_id}, {"shared_with": user_id}],
+                }
+            )
+
+            if not conversation:
+                logger.warning(
+                    f"Conversation not found or unauthorized - ID: {conversation_id}, User: {user_id}"
+                )
+                return None
+            conversation["_id"] = str(conversation["_id"])
+            return conversation
+        except Exception as e:
+            logger.error(f"Error fetching conversation: {str(e)}", exc_info=True)
+            return None
+
+    def save_conversation(
+        self,
+        conversation_id: Optional[str],
+        question: str,
+        response: str,
+        thought: str,
+        sources: List[Dict[str, Any]],
+        tool_calls: List[Dict[str, Any]],
+        llm: Any,
+        model_id: str,
+        decoded_token: Dict[str, Any],
+        index: Optional[int] = None,
+        api_key: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        is_shared_usage: bool = False,
+        shared_token: Optional[str] = None,
+        attachment_ids: Optional[List[str]] = None,
+    ) -> str:
+        """Save or update a conversation in the database"""
+        user_id = decoded_token.get("sub")
+        if not user_id:
+            raise ValueError("User ID not found in token")
+        current_time = datetime.now(timezone.utc)
+
+        # clean up in sources array such that we save max 1k characters for text part
+        for source in sources:
+            if "text" in source and isinstance(source["text"], str):
+                source["text"] = source["text"][:1000]
+
+        if conversation_id is not None and index is not None:
+            # Update existing conversation with new query
+
+            result = self.conversations_collection.update_one(
+                {
+                    "_id": ObjectId(conversation_id),
+                    "user": user_id,
+                    f"queries.{index}": {"$exists": True},
+                },
+                {
+                    "$set": {
+                        f"queries.{index}.prompt": question,
+                        f"queries.{index}.response": response,
+                        f"queries.{index}.thought": thought,
+                        f"queries.{index}.sources": sources,
+                        f"queries.{index}.tool_calls": tool_calls,
+                        f"queries.{index}.timestamp": current_time,
+                        f"queries.{index}.attachments": attachment_ids,
+                        f"queries.{index}.model_id": model_id,
+                    }
+                },
+            )
+
+            if result.matched_count == 0:
+                raise ValueError("Conversation not found or unauthorized")
+            self.conversations_collection.update_one(
+                {
+                    "_id": ObjectId(conversation_id),
+                    "user": user_id,
+                    f"queries.{index}": {"$exists": True},
+                },
+                {"$push": {"queries": {"$each": [], "$slice": index + 1}}},
+            )
+            return conversation_id
+        elif conversation_id:
+            # Append new message to existing conversation
+
+            result = self.conversations_collection.update_one(
+                {"_id": ObjectId(conversation_id), "user": user_id},
+                {
+                    "$push": {
+                        "queries": {
+                            "prompt": question,
+                            "response": response,
+                            "thought": thought,
+                            "sources": sources,
+                            "tool_calls": tool_calls,
+                            "timestamp": current_time,
+                            "attachments": attachment_ids,
+                            "model_id": model_id,
+                        }
+                    }
+                },
+            )
+
+            if result.matched_count == 0:
+                raise ValueError("Conversation not found or unauthorized")
+            return conversation_id
+        else:
+            # Create new conversation
+
+            messages_summary = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful assistant that creates concise conversation titles. "
+                    "Summarize conversations in 3 words or less using the same language as the user.",
+                },
+                {
+                    "role": "user",
+                    "content": "Summarise following conversation in no more than 3 words, "
+                    "respond ONLY with the summary, use the same language as the "
+                    "user query \n\nUser: " + question + "\n\n" + "AI: " + response,
+                },
+            ]
+
+            completion = llm.gen(
+                model=model_id, messages=messages_summary, max_tokens=30
+            )
+
+            conversation_data = {
+                "user": user_id,
+                "date": current_time,
+                "name": completion,
+                "queries": [
+                    {
+                        "prompt": question,
+                        "response": response,
+                        "thought": thought,
+                        "sources": sources,
+                        "tool_calls": tool_calls,
+                        "timestamp": current_time,
+                        "attachments": attachment_ids,
+                        "model_id": model_id,
+                    }
+                ],
+            }
+
+            if api_key:
+                if agent_id:
+                    conversation_data["agent_id"] = agent_id
+                    if is_shared_usage:
+                        conversation_data["is_shared_usage"] = is_shared_usage
+                        conversation_data["shared_token"] = shared_token
+                agent = self.agents_collection.find_one({"key": api_key})
+                if agent:
+                    conversation_data["api_key"] = agent["key"]
+            result = self.conversations_collection.insert_one(conversation_data)
+            return str(result.inserted_id)
+
+    def update_compression_metadata(
+        self, conversation_id: str, compression_metadata: Dict[str, Any]
+    ) -> None:
+        """
+        Update conversation with compression metadata.
+
+        Uses $push with $slice to keep only the most recent compression points,
+        preventing unbounded array growth. Since each compression incorporates
+        previous compressions, older points become redundant.
+
+        Args:
+            conversation_id: Conversation ID
+            compression_metadata: Compression point data
+        """
+        try:
+            self.conversations_collection.update_one(
+                {"_id": ObjectId(conversation_id)},
+                {
+                    "$set": {
+                        "compression_metadata.is_compressed": True,
+                        "compression_metadata.last_compression_at": compression_metadata.get(
+                            "timestamp"
+                        ),
+                    },
+                    "$push": {
+                        "compression_metadata.compression_points": {
+                            "$each": [compression_metadata],
+                            "$slice": -settings.COMPRESSION_MAX_HISTORY_POINTS,
+                        }
+                    },
+                },
+            )
+            logger.info(
+                f"Updated compression metadata for conversation {conversation_id}"
+            )
+        except Exception as e:
+            logger.error(
+                f"Error updating compression metadata: {str(e)}", exc_info=True
+            )
+            raise
+
+    def append_compression_message(
+        self, conversation_id: str, compression_metadata: Dict[str, Any]
+    ) -> None:
+        """
+        Append a synthetic compression summary entry into the conversation history.
+        This makes the summary visible in the DB alongside normal queries.
+        """
+        try:
+            summary = compression_metadata.get("compressed_summary", "")
+            if not summary:
+                return
+            timestamp = compression_metadata.get("timestamp", datetime.now(timezone.utc))
+
+            self.conversations_collection.update_one(
+                {"_id": ObjectId(conversation_id)},
+                {
+                    "$push": {
+                        "queries": {
+                            "prompt": "[Context Compression Summary]",
+                            "response": summary,
+                            "thought": "",
+                            "sources": [],
+                            "tool_calls": [],
+                            "timestamp": timestamp,
+                            "attachments": [],
+                            "model_id": compression_metadata.get("model_used"),
+                        }
+                    }
+                },
+            )
+            logger.info(f"Appended compression summary to conversation {conversation_id}")
+        except Exception as e:
+            logger.error(
+                f"Error appending compression summary: {str(e)}", exc_info=True
+            )
+
+    def get_compression_metadata(
+        self, conversation_id: str
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Get compression metadata for a conversation.
+
+        Args:
+            conversation_id: Conversation ID
+
+        Returns:
+            Compression metadata dict or None
+        """
+        try:
+            conversation = self.conversations_collection.find_one(
+                {"_id": ObjectId(conversation_id)}, {"compression_metadata": 1}
+            )
+            return conversation.get("compression_metadata") if conversation else None
+        except Exception as e:
+            logger.error(
+                f"Error getting compression metadata: {str(e)}", exc_info=True
+            )
+            return None

application/api/answer/services/prompt_renderer.py

@@ -0,0 +1,97 @@
+import logging
+from typing import Any, Dict, Optional
+
+from application.templates.namespaces import NamespaceManager
+
+from application.templates.template_engine import TemplateEngine, TemplateRenderError
+
+logger = logging.getLogger(__name__)
+
+
+class PromptRenderer:
+    """Service for rendering prompts with dynamic context using namespaces"""
+
+    def __init__(self):
+        self.template_engine = TemplateEngine()
+        self.namespace_manager = NamespaceManager()
+
+    def render_prompt(
+        self,
+        prompt_content: str,
+        user_id: Optional[str] = None,
+        request_id: Optional[str] = None,
+        passthrough_data: Optional[Dict[str, Any]] = None,
+        docs: Optional[list] = None,
+        docs_together: Optional[str] = None,
+        tools_data: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ) -> str:
+        """
+        Render prompt with full context from all namespaces.
+
+        Args:
+            prompt_content: Raw prompt template string
+            user_id: Current user identifier
+            request_id: Unique request identifier
+            passthrough_data: Parameters from web request
+            docs: RAG retrieved documents
+            docs_together: Concatenated document content
+            tools_data: Pre-fetched tool results organized by tool name
+            **kwargs: Additional parameters for namespace builders
+
+        Returns:
+            Rendered prompt string with all variables substituted
+
+        Raises:
+            TemplateRenderError: If template rendering fails
+        """
+        if not prompt_content:
+            return ""
+
+        uses_template = self._uses_template_syntax(prompt_content)
+
+        if not uses_template:
+            return self._apply_legacy_substitutions(prompt_content, docs_together)
+
+        try:
+            context = self.namespace_manager.build_context(
+                user_id=user_id,
+                request_id=request_id,
+                passthrough_data=passthrough_data,
+                docs=docs,
+                docs_together=docs_together,
+                tools_data=tools_data,
+                **kwargs,
+            )
+
+            return self.template_engine.render(prompt_content, context)
+        except TemplateRenderError:
+            raise
+        except Exception as e:
+            error_msg = f"Prompt rendering failed: {str(e)}"
+            logger.error(error_msg)
+            raise TemplateRenderError(error_msg) from e
+
+    def _uses_template_syntax(self, prompt_content: str) -> bool:
+        """Check if prompt uses Jinja2 template syntax"""
+        return "{{" in prompt_content and "}}" in prompt_content
+
+    def _apply_legacy_substitutions(
+        self, prompt_content: str, docs_together: Optional[str] = None
+    ) -> str:
+        """
+        Apply backward-compatible substitutions for old prompt format.
+
+        Handles legacy {summaries} and {query} placeholders during transition period.
+        """
+        if docs_together:
+            prompt_content = prompt_content.replace("{summaries}", docs_together)
+        return prompt_content
+
+    def validate_template(self, prompt_content: str) -> bool:
+        """Validate prompt template syntax"""
+        return self.template_engine.validate_template(prompt_content)
+
+    def extract_variables(self, prompt_content: str) -> set[str]:
+        """Extract all variable names from prompt template"""
+        return self.template_engine.extract_variables(prompt_content)

application/api/answer/services/stream_processor.py

@@ -0,0 +1,755 @@
+import datetime
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Set
+
+from bson.dbref import DBRef
+
+from bson.objectid import ObjectId
+
+from application.agents.agent_creator import AgentCreator
+from application.api.answer.services.compression import CompressionOrchestrator
+from application.api.answer.services.compression.token_counter import TokenCounter
+from application.api.answer.services.conversation_service import ConversationService
+from application.api.answer.services.prompt_renderer import PromptRenderer
+from application.core.model_utils import (
+    get_api_key_for_provider,
+    get_default_model_id,
+    get_provider_from_model_id,
+    validate_model_id,
+)
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.retriever.retriever_creator import RetrieverCreator
+from application.utils import (
+    calculate_doc_token_budget,
+    limit_chat_history,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def get_prompt(prompt_id: str, prompts_collection=None) -> str:
+    """
+    Get a prompt by preset name or MongoDB ID
+    """
+    current_dir = Path(__file__).resolve().parents[3]
+    prompts_dir = current_dir / "prompts"
+
+    preset_mapping = {
+        "default": "chat_combine_default.txt",
+        "creative": "chat_combine_creative.txt",
+        "strict": "chat_combine_strict.txt",
+        "reduce": "chat_reduce_prompt.txt",
+    }
+
+    if prompt_id in preset_mapping:
+        file_path = os.path.join(prompts_dir, preset_mapping[prompt_id])
+        try:
+            with open(file_path, "r") as f:
+                return f.read()
+        except FileNotFoundError:
+            raise FileNotFoundError(f"Prompt file not found: {file_path}")
+    try:
+        if prompts_collection is None:
+            mongo = MongoDB.get_client()
+            db = mongo[settings.MONGO_DB_NAME]
+            prompts_collection = db["prompts"]
+        prompt_doc = prompts_collection.find_one({"_id": ObjectId(prompt_id)})
+        if not prompt_doc:
+            raise ValueError(f"Prompt with ID {prompt_id} not found")
+        return prompt_doc["content"]
+    except Exception as e:
+        raise ValueError(f"Invalid prompt ID: {prompt_id}") from e
+
+
+class StreamProcessor:
+    def __init__(
+        self, request_data: Dict[str, Any], decoded_token: Optional[Dict[str, Any]]
+    ):
+        mongo = MongoDB.get_client()
+        self.db = mongo[settings.MONGO_DB_NAME]
+        self.agents_collection = self.db["agents"]
+        self.attachments_collection = self.db["attachments"]
+        self.prompts_collection = self.db["prompts"]
+
+        self.data = request_data
+        self.decoded_token = decoded_token
+        self.initial_user_id = (
+            self.decoded_token.get("sub") if self.decoded_token is not None else None
+        )
+        self.conversation_id = self.data.get("conversation_id")
+        self.source = {}
+        self.all_sources = []
+        self.attachments = []
+        self.history = []
+        self.retrieved_docs = []
+        self.agent_config = {}
+        self.retriever_config = {}
+        self.is_shared_usage = False
+        self.shared_token = None
+        self.model_id: Optional[str] = None
+        self.conversation_service = ConversationService()
+        self.compression_orchestrator = CompressionOrchestrator(
+            self.conversation_service
+        )
+        self.prompt_renderer = PromptRenderer()
+        self._prompt_content: Optional[str] = None
+        self._required_tool_actions: Optional[Dict[str, Set[Optional[str]]]] = None
+        self.compressed_summary: Optional[str] = None
+        self.compressed_summary_tokens: int = 0
+
+    def initialize(self):
+        """Initialize all required components for processing"""
+        self._configure_agent()
+        self._validate_and_set_model()
+        self._configure_source()
+        self._configure_retriever()
+        self._load_conversation_history()
+        self._process_attachments()
+
+    def _load_conversation_history(self):
+        """Load conversation history either from DB or request"""
+        if self.conversation_id and self.initial_user_id:
+            conversation = self.conversation_service.get_conversation(
+                self.conversation_id, self.initial_user_id
+            )
+            if not conversation:
+                raise ValueError("Conversation not found or unauthorized")
+
+            # Check if compression is enabled and needed
+            if settings.ENABLE_CONVERSATION_COMPRESSION:
+                self._handle_compression(conversation)
+            else:
+                # Original behavior - load all history
+                self.history = [
+                    {"prompt": query["prompt"], "response": query["response"]}
+                    for query in conversation.get("queries", [])
+                ]
+        else:
+            self.history = limit_chat_history(
+                json.loads(self.data.get("history", "[]")), model_id=self.model_id
+            )
+
+    def _handle_compression(self, conversation: Dict[str, Any]):
+        """
+        Handle conversation compression logic using orchestrator.
+
+        Args:
+            conversation: Full conversation document
+        """
+        try:
+            # Use orchestrator to handle all compression logic
+            result = self.compression_orchestrator.compress_if_needed(
+                conversation_id=self.conversation_id,
+                user_id=self.initial_user_id,
+                model_id=self.model_id,
+                decoded_token=self.decoded_token,
+            )
+
+            if not result.success:
+                logger.error(
+                    f"Compression failed: {result.error}, using full history"
+                )
+                self.history = [
+                    {"prompt": query["prompt"], "response": query["response"]}
+                    for query in conversation.get("queries", [])
+                ]
+                return
+
+            # Set compressed summary if compression was performed
+            if result.compression_performed and result.compressed_summary:
+                self.compressed_summary = result.compressed_summary
+                self.compressed_summary_tokens = TokenCounter.count_message_tokens(
+                    [{"content": result.compressed_summary}]
+                )
+                logger.info(
+                    f"Using compressed summary ({self.compressed_summary_tokens} tokens) "
+                    f"+ {len(result.recent_queries)} recent messages"
+                )
+
+            # Build history from recent queries
+            self.history = result.as_history()
+
+        except Exception as e:
+            logger.error(
+                f"Error handling compression, falling back to standard history: {str(e)}",
+                exc_info=True,
+            )
+            # Fallback to original behavior
+            self.history = [
+                {"prompt": query["prompt"], "response": query["response"]}
+                for query in conversation.get("queries", [])
+            ]
+
+    def _process_attachments(self):
+        """Process any attachments in the request"""
+        attachment_ids = self.data.get("attachments", [])
+        self.attachments = self._get_attachments_content(
+            attachment_ids, self.initial_user_id
+        )
+
+    def _get_attachments_content(self, attachment_ids, user_id):
+        """
+        Retrieve content from attachment documents based on their IDs.
+        """
+        if not attachment_ids:
+            return []
+        attachments = []
+        for attachment_id in attachment_ids:
+            try:
+                attachment_doc = self.attachments_collection.find_one(
+                    {"_id": ObjectId(attachment_id), "user": user_id}
+                )
+
+                if attachment_doc:
+                    attachments.append(attachment_doc)
+            except Exception as e:
+                logger.error(
+                    f"Error retrieving attachment {attachment_id}: {e}", exc_info=True
+                )
+        return attachments
+
+    def _validate_and_set_model(self):
+        """Validate and set model_id from request"""
+        from application.core.model_settings import ModelRegistry
+
+        requested_model = self.data.get("model_id")
+
+        if requested_model:
+            if not validate_model_id(requested_model):
+                registry = ModelRegistry.get_instance()
+                available_models = [m.id for m in registry.get_enabled_models()]
+                raise ValueError(
+                    f"Invalid model_id '{requested_model}'. "
+                    f"Available models: {', '.join(available_models[:5])}"
+                    + (f" and {len(available_models) - 5} more" if len(available_models) > 5 else "")
+                )
+            self.model_id = requested_model
+        else:
+            # Check if agent has a default model configured
+            agent_default_model = self.agent_config.get("default_model_id", "")
+            if agent_default_model and validate_model_id(agent_default_model):
+                self.model_id = agent_default_model
+            else:
+                self.model_id = get_default_model_id()
+
+    def _get_agent_key(self, agent_id: Optional[str], user_id: Optional[str]) -> tuple:
+        """Get API key for agent with access control"""
+        if not agent_id:
+            return None, False, None
+        try:
+            agent = self.agents_collection.find_one({"_id": ObjectId(agent_id)})
+            if agent is None:
+                raise Exception("Agent not found")
+            is_owner = agent.get("user") == user_id
+            is_shared_with_user = agent.get(
+                "shared_publicly", False
+            ) or user_id in agent.get("shared_with", [])
+
+            if not (is_owner or is_shared_with_user):
+                raise Exception("Unauthorized access to the agent")
+            if is_owner:
+                self.agents_collection.update_one(
+                    {"_id": ObjectId(agent_id)},
+                    {
+                        "$set": {
+                            "lastUsedAt": datetime.datetime.now(datetime.timezone.utc)
+                        }
+                    },
+                )
+            return str(agent["key"]), not is_owner, agent.get("shared_token")
+        except Exception as e:
+            logger.error(f"Error in get_agent_key: {str(e)}", exc_info=True)
+            raise
+
+    def _get_data_from_api_key(self, api_key: str) -> Dict[str, Any]:
+        data = self.agents_collection.find_one({"key": api_key})
+        if not data:
+            raise Exception("Invalid API Key, please generate a new key", 401)
+        source = data.get("source")
+        if isinstance(source, DBRef):
+            source_doc = self.db.dereference(source)
+            if source_doc:
+                data["source"] = str(source_doc["_id"])
+                data["retriever"] = source_doc.get("retriever", data.get("retriever"))
+                data["chunks"] = source_doc.get("chunks", data.get("chunks"))
+            else:
+                data["source"] = None
+        elif source == "default":
+            data["source"] = "default"
+        else:
+            data["source"] = None
+        # Handle multiple sources
+
+        sources = data.get("sources", [])
+        if sources and isinstance(sources, list):
+            sources_list = []
+            for i, source_ref in enumerate(sources):
+                if source_ref == "default":
+                    processed_source = {
+                        "id": "default",
+                        "retriever": "classic",
+                        "chunks": data.get("chunks", "2"),
+                    }
+                    sources_list.append(processed_source)
+                elif isinstance(source_ref, DBRef):
+                    source_doc = self.db.dereference(source_ref)
+                    if source_doc:
+                        processed_source = {
+                            "id": str(source_doc["_id"]),
+                            "retriever": source_doc.get("retriever", "classic"),
+                            "chunks": source_doc.get("chunks", data.get("chunks", "2")),
+                        }
+                        sources_list.append(processed_source)
+            data["sources"] = sources_list
+        else:
+            data["sources"] = []
+
+        # Preserve model configuration from agent
+        data["default_model_id"] = data.get("default_model_id", "")
+
+        return data
+
+    def _configure_source(self):
+        """Configure the source based on agent data"""
+        api_key = self.data.get("api_key") or self.agent_key
+
+        if api_key:
+            agent_data = self._get_data_from_api_key(api_key)
+
+            if agent_data.get("sources") and len(agent_data["sources"]) > 0:
+                source_ids = [
+                    source["id"] for source in agent_data["sources"] if source.get("id")
+                ]
+                if source_ids:
+                    self.source = {"active_docs": source_ids}
+                else:
+                    self.source = {}
+                self.all_sources = agent_data["sources"]
+            elif agent_data.get("source"):
+                self.source = {"active_docs": agent_data["source"]}
+                self.all_sources = [
+                    {
+                        "id": agent_data["source"],
+                        "retriever": agent_data.get("retriever", "classic"),
+                    }
+                ]
+            else:
+                self.source = {}
+                self.all_sources = []
+            return
+        if "active_docs" in self.data:
+            self.source = {"active_docs": self.data["active_docs"]}
+            return
+        self.source = {}
+        self.all_sources = []
+
+    def _configure_agent(self):
+        """Configure the agent based on request data"""
+        agent_id = self.data.get("agent_id")
+        self.agent_key, self.is_shared_usage, self.shared_token = self._get_agent_key(
+            agent_id, self.initial_user_id
+        )
+
+        api_key = self.data.get("api_key")
+        if api_key:
+            data_key = self._get_data_from_api_key(api_key)
+            self.agent_config.update(
+                {
+                    "prompt_id": data_key.get("prompt_id", "default"),
+                    "agent_type": data_key.get("agent_type", settings.AGENT_NAME),
+                    "user_api_key": api_key,
+                    "json_schema": data_key.get("json_schema"),
+                    "default_model_id": data_key.get("default_model_id", ""),
+                }
+            )
+            self.initial_user_id = data_key.get("user")
+            self.decoded_token = {"sub": data_key.get("user")}
+            if data_key.get("source"):
+                self.source = {"active_docs": data_key["source"]}
+            if data_key.get("retriever"):
+                self.retriever_config["retriever_name"] = data_key["retriever"]
+            if data_key.get("chunks") is not None:
+                try:
+                    self.retriever_config["chunks"] = int(data_key["chunks"])
+                except (ValueError, TypeError):
+                    logger.warning(
+                        f"Invalid chunks value: {data_key['chunks']}, using default value 2"
+                    )
+                    self.retriever_config["chunks"] = 2
+        elif self.agent_key:
+            data_key = self._get_data_from_api_key(self.agent_key)
+            self.agent_config.update(
+                {
+                    "prompt_id": data_key.get("prompt_id", "default"),
+                    "agent_type": data_key.get("agent_type", settings.AGENT_NAME),
+                    "user_api_key": self.agent_key,
+                    "json_schema": data_key.get("json_schema"),
+                    "default_model_id": data_key.get("default_model_id", ""),
+                }
+            )
+            self.decoded_token = (
+                self.decoded_token
+                if self.is_shared_usage
+                else {"sub": data_key.get("user")}
+            )
+            if data_key.get("source"):
+                self.source = {"active_docs": data_key["source"]}
+            if data_key.get("retriever"):
+                self.retriever_config["retriever_name"] = data_key["retriever"]
+            if data_key.get("chunks") is not None:
+                try:
+                    self.retriever_config["chunks"] = int(data_key["chunks"])
+                except (ValueError, TypeError):
+                    logger.warning(
+                        f"Invalid chunks value: {data_key['chunks']}, using default value 2"
+                    )
+                    self.retriever_config["chunks"] = 2
+        else:
+            self.agent_config.update(
+                {
+                    "prompt_id": self.data.get("prompt_id", "default"),
+                    "agent_type": settings.AGENT_NAME,
+                    "user_api_key": None,
+                    "json_schema": None,
+                    "default_model_id": "",
+                }
+            )
+
+    def _configure_retriever(self):
+        history_token_limit = int(self.data.get("token_limit", 2000))
+        doc_token_limit = calculate_doc_token_budget(
+            model_id=self.model_id, history_token_limit=history_token_limit
+        )
+
+        self.retriever_config = {
+            "retriever_name": self.data.get("retriever", "classic"),
+            "chunks": int(self.data.get("chunks", 2)),
+            "doc_token_limit": doc_token_limit,
+            "history_token_limit": history_token_limit,
+        }
+
+        api_key = self.data.get("api_key") or self.agent_key
+        if not api_key and "isNoneDoc" in self.data and self.data["isNoneDoc"]:
+            self.retriever_config["chunks"] = 0
+
+    def create_retriever(self):
+        return RetrieverCreator.create_retriever(
+            self.retriever_config["retriever_name"],
+            source=self.source,
+            chat_history=self.history,
+            prompt=get_prompt(self.agent_config["prompt_id"], self.prompts_collection),
+            chunks=self.retriever_config["chunks"],
+            doc_token_limit=self.retriever_config.get("doc_token_limit", 50000),
+            model_id=self.model_id,
+            user_api_key=self.agent_config["user_api_key"],
+            decoded_token=self.decoded_token,
+        )
+
+    def pre_fetch_docs(self, question: str) -> tuple[Optional[str], Optional[list]]:
+        """Pre-fetch documents for template rendering before agent creation"""
+        if self.data.get("isNoneDoc", False):
+            logger.info("Pre-fetch skipped: isNoneDoc=True")
+            return None, None
+        try:
+            retriever = self.create_retriever()
+            logger.info(
+                f"Pre-fetching docs with chunks={retriever.chunks}, doc_token_limit={retriever.doc_token_limit}"
+            )
+            docs = retriever.search(question)
+            logger.info(f"Pre-fetch retrieved {len(docs) if docs else 0} documents")
+
+            if not docs:
+                logger.info("Pre-fetch: No documents returned from search")
+                return None, None
+            self.retrieved_docs = docs
+
+            docs_with_filenames = []
+            for doc in docs:
+                filename = doc.get("filename") or doc.get("title") or doc.get("source")
+                if filename:
+                    chunk_header = str(filename)
+                    docs_with_filenames.append(f"{chunk_header}\n{doc['text']}")
+                else:
+                    docs_with_filenames.append(doc["text"])
+            docs_together = "\n\n".join(docs_with_filenames)
+
+            logger.info(f"Pre-fetch docs_together size: {len(docs_together)} chars")
+
+            return docs_together, docs
+        except Exception as e:
+            logger.error(f"Failed to pre-fetch docs: {str(e)}", exc_info=True)
+            return None, None
+
+    def pre_fetch_tools(self) -> Optional[Dict[str, Any]]:
+        """Pre-fetch tool data for template rendering before agent creation
+
+        Can be controlled via:
+        1. Global setting: ENABLE_TOOL_PREFETCH in .env
+        2. Per-request: disable_tool_prefetch in request data
+        """
+        if not settings.ENABLE_TOOL_PREFETCH:
+            logger.info(
+                "Tool pre-fetching disabled globally via ENABLE_TOOL_PREFETCH setting"
+            )
+            return None
+
+        if self.data.get("disable_tool_prefetch", False):
+            logger.info("Tool pre-fetching disabled for this request")
+            return None
+
+        required_tool_actions = self._get_required_tool_actions()
+        filtering_enabled = required_tool_actions is not None
+
+        try:
+            user_tools_collection = self.db["user_tools"]
+            user_id = self.initial_user_id or "local"
+
+            user_tools = list(
+                user_tools_collection.find({"user": user_id, "status": True})
+            )
+
+            if not user_tools:
+                return None
+
+            tools_data = {}
+
+            for tool_doc in user_tools:
+                tool_name = tool_doc.get("name")
+                tool_id = str(tool_doc.get("_id"))
+
+                if filtering_enabled:
+                    required_actions_by_name = required_tool_actions.get(
+                        tool_name, set()
+                    )
+                    required_actions_by_id = required_tool_actions.get(tool_id, set())
+
+                    required_actions = required_actions_by_name | required_actions_by_id
+
+                    if not required_actions:
+                        continue
+                else:
+                    required_actions = None
+
+                tool_data = self._fetch_tool_data(tool_doc, required_actions)
+                if tool_data:
+                    tools_data[tool_name] = tool_data
+                    tools_data[tool_id] = tool_data
+
+            return tools_data if tools_data else None
+        except Exception as e:
+            logger.warning(f"Failed to pre-fetch tools: {type(e).__name__}")
+            return None
+
+    def _fetch_tool_data(
+        self,
+        tool_doc: Dict[str, Any],
+        required_actions: Optional[Set[Optional[str]]],
+    ) -> Optional[Dict[str, Any]]:
+        """Fetch and execute tool actions with saved parameters"""
+        try:
+            from application.agents.tools.tool_manager import ToolManager
+
+            tool_name = tool_doc.get("name")
+            tool_config = tool_doc.get("config", {}).copy()
+            tool_config["tool_id"] = str(tool_doc["_id"])
+
+            tool_manager = ToolManager(config={tool_name: tool_config})
+            user_id = self.initial_user_id or "local"
+            tool = tool_manager.load_tool(tool_name, tool_config, user_id=user_id)
+
+            if not tool:
+                logger.debug(f"Tool '{tool_name}' failed to load")
+                return None
+
+            tool_actions = tool.get_actions_metadata()
+            if not tool_actions:
+                logger.debug(f"Tool '{tool_name}' has no actions")
+                return None
+
+            saved_actions = tool_doc.get("actions", [])
+
+            include_all_actions = required_actions is None or (
+                required_actions and None in required_actions
+            )
+            allowed_actions: Set[str] = (
+                {action for action in required_actions if isinstance(action, str)}
+                if required_actions
+                else set()
+            )
+
+            action_results = {}
+            for action_meta in tool_actions:
+                action_name = action_meta.get("name")
+                if action_name is None:
+                    continue
+                if (
+                    not include_all_actions
+                    and allowed_actions
+                    and action_name not in allowed_actions
+                ):
+                    continue
+
+                try:
+                    saved_action = None
+                    for sa in saved_actions:
+                        if sa.get("name") == action_name:
+                            saved_action = sa
+                            break
+
+                    action_params = action_meta.get("parameters", {})
+                    properties = action_params.get("properties", {})
+
+                    kwargs = {}
+                    for param_name, param_spec in properties.items():
+                        if saved_action:
+                            saved_props = saved_action.get("parameters", {}).get(
+                                "properties", {}
+                            )
+                            if param_name in saved_props:
+                                param_value = saved_props[param_name].get("value")
+                                if param_value is not None:
+                                    kwargs[param_name] = param_value
+                                    continue
+
+                        if param_name in tool_config:
+                            kwargs[param_name] = tool_config[param_name]
+                        elif "default" in param_spec:
+                            kwargs[param_name] = param_spec["default"]
+
+                    result = tool.execute_action(action_name, **kwargs)
+                    action_results[action_name] = result
+                except Exception as e:
+                    logger.debug(
+                        f"Action '{action_name}' execution failed: {type(e).__name__}"
+                    )
+                    continue
+
+            return action_results if action_results else None
+
+        except Exception as e:
+            logger.debug(f"Tool pre-fetch failed for '{tool_name}': {type(e).__name__}")
+            return None
+
+    def _get_prompt_content(self) -> Optional[str]:
+        """Retrieve and cache the raw prompt content for the current agent configuration."""
+        if self._prompt_content is not None:
+            return self._prompt_content
+        prompt_id = (
+            self.agent_config.get("prompt_id")
+            if isinstance(self.agent_config, dict)
+            else None
+        )
+        if not prompt_id:
+            return None
+        try:
+            self._prompt_content = get_prompt(prompt_id, self.prompts_collection)
+        except ValueError as e:
+            logger.debug(f"Invalid prompt ID '{prompt_id}': {str(e)}")
+            self._prompt_content = None
+        except Exception as e:
+            logger.debug(f"Failed to fetch prompt '{prompt_id}': {type(e).__name__}")
+            self._prompt_content = None
+        return self._prompt_content
+
+    def _get_required_tool_actions(self) -> Optional[Dict[str, Set[Optional[str]]]]:
+        """Determine which tool actions are referenced in the prompt template"""
+        if self._required_tool_actions is not None:
+            return self._required_tool_actions
+
+        prompt_content = self._get_prompt_content()
+        if prompt_content is None:
+            return None
+
+        if "{{" not in prompt_content or "}}" not in prompt_content:
+            self._required_tool_actions = {}
+            return self._required_tool_actions
+
+        try:
+            from application.templates.template_engine import TemplateEngine
+
+            template_engine = TemplateEngine()
+            usages = template_engine.extract_tool_usages(prompt_content)
+            self._required_tool_actions = usages
+            return self._required_tool_actions
+        except Exception as e:
+            logger.debug(f"Failed to extract tool usages: {type(e).__name__}")
+            self._required_tool_actions = {}
+            return self._required_tool_actions
+
+    def _fetch_memory_tool_data(
+        self, tool_doc: Dict[str, Any]
+    ) -> Optional[Dict[str, Any]]:
+        """Fetch memory tool data for pre-injection into prompt"""
+        try:
+            tool_config = tool_doc.get("config", {}).copy()
+            tool_config["tool_id"] = str(tool_doc["_id"])
+
+            from application.agents.tools.memory import MemoryTool
+
+            memory_tool = MemoryTool(tool_config, self.initial_user_id)
+
+            root_view = memory_tool.execute_action("view", path="/")
+
+            if "Error:" in root_view or not root_view.strip():
+                return None
+
+            return {"root": root_view, "available": True}
+        except Exception as e:
+            logger.warning(f"Failed to fetch memory tool data: {str(e)}")
+            return None
+
+    def create_agent(
+        self,
+        docs_together: Optional[str] = None,
+        docs: Optional[list] = None,
+        tools_data: Optional[Dict[str, Any]] = None,
+    ):
+        """Create and return the configured agent with rendered prompt"""
+        raw_prompt = self._get_prompt_content()
+        if raw_prompt is None:
+            raw_prompt = get_prompt(
+                self.agent_config["prompt_id"], self.prompts_collection
+            )
+            self._prompt_content = raw_prompt
+
+        rendered_prompt = self.prompt_renderer.render_prompt(
+            prompt_content=raw_prompt,
+            user_id=self.initial_user_id,
+            request_id=self.data.get("request_id"),
+            passthrough_data=self.data.get("passthrough"),
+            docs=docs,
+            docs_together=docs_together,
+            tools_data=tools_data,
+        )
+
+        provider = (
+            get_provider_from_model_id(self.model_id)
+            if self.model_id
+            else settings.LLM_PROVIDER
+        )
+        system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER)
+
+        agent = AgentCreator.create_agent(
+            self.agent_config["agent_type"],
+            endpoint="stream",
+            llm_name=provider or settings.LLM_PROVIDER,
+            model_id=self.model_id,
+            api_key=system_api_key,
+            user_api_key=self.agent_config["user_api_key"],
+            prompt=rendered_prompt,
+            chat_history=self.history,
+            retrieved_docs=self.retrieved_docs,
+            decoded_token=self.decoded_token,
+            attachments=self.attachments,
+            json_schema=self.agent_config.get("json_schema"),
+            compressed_summary=self.compressed_summary,
+        )
+
+        agent.conversation_id = self.conversation_id
+        agent.initial_user_id = self.initial_user_id
+
+        return agent

application/api/connector/routes.py

@@ -0,0 +1,489 @@
+import base64
+import datetime
+import json
+import uuid
+
+
+from bson.objectid import ObjectId
+from flask import (
+    Blueprint,
+    current_app,
+    jsonify,
+    make_response,
+    request
+)
+from flask_restx import fields, Namespace, Resource
+
+
+from application.api.user.tasks import (
+    ingest_connector_task,
+)
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.api import api
+
+
+from application.parser.connectors.connector_creator import ConnectorCreator
+
+
+mongo = MongoDB.get_client()
+db = mongo[settings.MONGO_DB_NAME]
+sources_collection = db["sources"]
+sessions_collection = db["connector_sessions"]
+
+connector = Blueprint("connector", __name__)
+connectors_ns = Namespace("connectors", description="Connector operations", path="/")
+api.add_namespace(connectors_ns)
+
+
+
+@connectors_ns.route("/api/connectors/auth")
+class ConnectorAuth(Resource):
+    @api.doc(description="Get connector OAuth authorization URL", params={"provider": "Connector provider (e.g., google_drive)"})
+    def get(self):
+        try:
+            provider = request.args.get('provider') or request.args.get('source')
+            if not provider:
+                return make_response(jsonify({"success": False, "error": "Missing provider"}), 400)
+
+            if not ConnectorCreator.is_supported(provider):
+                return make_response(jsonify({"success": False, "error": f"Unsupported provider: {provider}"}), 400)
+
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+            user_id = decoded_token.get('sub')
+
+            now = datetime.datetime.now(datetime.timezone.utc)
+            result = sessions_collection.insert_one({
+                "provider": provider,
+                "user": user_id,
+                "status": "pending",
+                "created_at": now
+            })
+            state_dict = {
+                "provider": provider,
+                "object_id": str(result.inserted_id)
+            }
+            state = base64.urlsafe_b64encode(json.dumps(state_dict).encode()).decode()
+
+            auth = ConnectorCreator.create_auth(provider)
+            authorization_url = auth.get_authorization_url(state=state)
+            return make_response(jsonify({
+                "success": True,
+                "authorization_url": authorization_url,
+                "state": state
+            }), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error generating connector auth URL: {e}")
+            return make_response(jsonify({"success": False, "error": str(e)}), 500)
+
+
+@connectors_ns.route("/api/connectors/callback")
+class ConnectorsCallback(Resource):
+    @api.doc(description="Handle OAuth callback for external connectors")
+    def get(self):
+        """Handle OAuth callback for external connectors"""
+        try:
+            from application.parser.connectors.connector_creator import ConnectorCreator
+            from flask import request, redirect
+
+            authorization_code = request.args.get('code')
+            state = request.args.get('state')
+            error = request.args.get('error')
+
+            state_dict = json.loads(base64.urlsafe_b64decode(state.encode()).decode())
+            provider = state_dict["provider"]
+            state_object_id = state_dict["object_id"]
+
+            if error:
+                if error == "access_denied":
+                    return redirect(f"/api/connectors/callback-status?status=cancelled&message=Authentication+was+cancelled.+You+can+try+again+if+you'd+like+to+connect+your+account.&provider={provider}")
+                else:
+                    current_app.logger.warning(f"OAuth error in callback: {error}")
+                    return redirect(f"/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.&provider={provider}")
+
+            if not authorization_code:
+                return redirect(f"/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.&provider={provider}")
+
+            try:
+                auth = ConnectorCreator.create_auth(provider)
+                token_info = auth.exchange_code_for_tokens(authorization_code)
+
+                session_token = str(uuid.uuid4())
+
+                try:
+                    credentials = auth.create_credentials_from_token_info(token_info)
+                    service = auth.build_drive_service(credentials)
+                    user_info = service.about().get(fields="user").execute()
+                    user_email = user_info.get('user', {}).get('emailAddress', 'Connected User')
+                except Exception as e:
+                    current_app.logger.warning(f"Could not get user info: {e}")
+                    user_email = 'Connected User'
+
+                sanitized_token_info = {
+                    "access_token": token_info.get("access_token"),
+                    "refresh_token": token_info.get("refresh_token"),
+                    "token_uri": token_info.get("token_uri"),
+                    "expiry": token_info.get("expiry")
+                }
+
+                sessions_collection.find_one_and_update(
+                    {"_id": ObjectId(state_object_id), "provider": provider},
+                    {
+                        "$set": {
+                            "session_token": session_token,
+                            "token_info": sanitized_token_info,
+                            "user_email": user_email,
+                            "status": "authorized"
+                        }
+                    }
+                )
+
+                # Redirect to success page with session token and user email
+                return redirect(f"/api/connectors/callback-status?status=success&message=Authentication+successful&provider={provider}&session_token={session_token}&user_email={user_email}")
+
+            except Exception as e:
+                current_app.logger.error(f"Error exchanging code for tokens: {str(e)}", exc_info=True)
+                return redirect(f"/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.&provider={provider}")
+
+        except Exception as e:
+            current_app.logger.error(f"Error handling connector callback: {e}")
+            return redirect("/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.")
+
+
+@connectors_ns.route("/api/connectors/files")
+class ConnectorFiles(Resource):
+    @api.expect(api.model("ConnectorFilesModel", {
+        "provider": fields.String(required=True), 
+        "session_token": fields.String(required=True), 
+        "folder_id": fields.String(required=False), 
+        "limit": fields.Integer(required=False), 
+        "page_token": fields.String(required=False),
+        "search_query": fields.String(required=False)
+    }))
+    @api.doc(description="List files from a connector provider (supports pagination and search)")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            session_token = data.get('session_token')
+            folder_id = data.get('folder_id')
+            limit = data.get('limit', 10)
+            page_token = data.get('page_token')
+            search_query = data.get('search_query')
+            
+            if not provider or not session_token:
+                return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400)
+
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+            user = decoded_token.get('sub')
+            session = sessions_collection.find_one({"session_token": session_token, "user": user})
+            if not session:
+                return make_response(jsonify({"success": False, "error": "Invalid or unauthorized session"}), 401)
+
+            loader = ConnectorCreator.create_connector(provider, session_token)
+            input_config = {
+                'limit': limit,
+                'list_only': True,
+                'session_token': session_token,
+                'folder_id': folder_id,
+                'page_token': page_token
+            }
+            if search_query:
+                input_config['search_query'] = search_query
+                
+            documents = loader.load_data(input_config)
+
+            files = []
+            for doc in documents[:limit]:
+                metadata = doc.extra_info
+                modified_time = metadata.get('modified_time')
+                if modified_time:
+                    date_part = modified_time.split('T')[0]
+                    time_part = modified_time.split('T')[1].split('.')[0].split('Z')[0]
+                    formatted_time = f"{date_part} {time_part}"
+                else:
+                    formatted_time = None
+
+                files.append({
+                    'id': doc.doc_id,
+                    'name': metadata.get('file_name', 'Unknown File'),
+                    'type': metadata.get('mime_type', 'unknown'),
+                    'size': metadata.get('size', None),
+                    'modifiedTime': formatted_time,
+                    'isFolder': metadata.get('is_folder', False)
+                })
+
+            next_token = getattr(loader, 'next_page_token', None)
+            has_more = bool(next_token)
+
+            return make_response(jsonify({
+                "success": True, 
+                "files": files, 
+                "total": len(files), 
+                "next_page_token": next_token, 
+                "has_more": has_more
+            }), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error loading connector files: {e}")
+            return make_response(jsonify({"success": False, "error": f"Failed to load files: {str(e)}"}), 500)
+
+
+@connectors_ns.route("/api/connectors/validate-session")
+class ConnectorValidateSession(Resource):
+    @api.expect(api.model("ConnectorValidateSessionModel", {"provider": fields.String(required=True), "session_token": fields.String(required=True)}))
+    @api.doc(description="Validate connector session token and return user info and access token")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            session_token = data.get('session_token')
+            if not provider or not session_token:
+                return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400)
+
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+            user = decoded_token.get('sub')
+
+            session = sessions_collection.find_one({"session_token": session_token, "user": user})
+            if not session or "token_info" not in session:
+                return make_response(jsonify({"success": False, "error": "Invalid or expired session"}), 401)
+
+            token_info = session["token_info"]
+            auth = ConnectorCreator.create_auth(provider)
+            is_expired = auth.is_token_expired(token_info)
+
+            if is_expired and token_info.get('refresh_token'):
+                try:
+                    refreshed_token_info = auth.refresh_access_token(token_info.get('refresh_token'))
+                    sanitized_token_info = {
+                    "access_token": refreshed_token_info.get("access_token"),
+                    "refresh_token": refreshed_token_info.get("refresh_token"),
+                    "token_uri": refreshed_token_info.get("token_uri"),
+                    "expiry": refreshed_token_info.get("expiry")
+                }    
+                    sessions_collection.update_one(
+                        {"session_token": session_token},
+                        {"$set": {"token_info": sanitized_token_info}}
+                    )
+                    token_info = sanitized_token_info
+                    is_expired = False
+                except Exception as refresh_error:
+                    current_app.logger.error(f"Failed to refresh token: {refresh_error}")
+            
+            if is_expired:
+                return make_response(jsonify({
+                    "success": False,
+                    "expired": True,
+                    "error": "Session token has expired. Please reconnect."
+                }), 401)
+
+            return make_response(jsonify({
+                "success": True,
+                "expired": False,
+                "user_email": session.get('user_email', 'Connected User'),
+                "access_token": token_info.get('access_token')
+            }), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error validating connector session: {e}")
+            return make_response(jsonify({"success": False, "error": str(e)}), 500)
+
+
+@connectors_ns.route("/api/connectors/disconnect")
+class ConnectorDisconnect(Resource):
+    @api.expect(api.model("ConnectorDisconnectModel", {"provider": fields.String(required=True), "session_token": fields.String(required=False)}))
+    @api.doc(description="Disconnect a connector session")
+    def post(self):
+        try:
+            data = request.get_json()
+            provider = data.get('provider')
+            session_token = data.get('session_token')
+            if not provider:
+                return make_response(jsonify({"success": False, "error": "provider is required"}), 400)
+
+
+            if session_token:
+                sessions_collection.delete_one({"session_token": session_token})
+            
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as e:
+            current_app.logger.error(f"Error disconnecting connector session: {e}")
+            return make_response(jsonify({"success": False, "error": str(e)}), 500)
+
+
+@connectors_ns.route("/api/connectors/sync")
+class ConnectorSync(Resource):
+    @api.expect(
+        api.model(
+            "ConnectorSyncModel",
+            {
+                "source_id": fields.String(required=True, description="Source ID to sync"),
+                "session_token": fields.String(required=True, description="Authentication token")
+            },
+        )
+    )
+    @api.doc(description="Sync connector source to check for modifications")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+
+        try:
+            data = request.get_json()
+            source_id = data.get('source_id')
+            session_token = data.get('session_token')
+
+            if not all([source_id, session_token]):
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "source_id and session_token are required"
+                    }), 
+                    400
+                )
+            source = sources_collection.find_one({"_id": ObjectId(source_id)})
+            if not source:
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "Source not found"
+                    }), 
+                    404
+                )
+
+            if source.get('user') != decoded_token.get('sub'):
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "Unauthorized access to source"
+                    }), 
+                    403
+                )
+
+            remote_data = {}
+            try:
+                if source.get('remote_data'):
+                    remote_data = json.loads(source.get('remote_data'))
+            except json.JSONDecodeError:
+                current_app.logger.error(f"Invalid remote_data format for source {source_id}")
+                remote_data = {}
+
+            source_type = remote_data.get('provider')
+            if not source_type:
+                return make_response(
+                    jsonify({
+                        "success": False,
+                        "error": "Source provider not found in remote_data"
+                    }), 
+                    400
+                )
+
+            # Extract configuration from remote_data
+            file_ids = remote_data.get('file_ids', [])
+            folder_ids = remote_data.get('folder_ids', [])
+            recursive = remote_data.get('recursive', True)
+
+            # Start the sync task
+            task = ingest_connector_task.delay(
+                job_name=source.get('name'),
+                user=decoded_token.get('sub'),
+                source_type=source_type,
+                session_token=session_token,
+                file_ids=file_ids,
+                folder_ids=folder_ids,
+                recursive=recursive,
+                retriever=source.get('retriever', 'classic'),
+                operation_mode="sync",
+                doc_id=source_id,
+                sync_frequency=source.get('sync_frequency', 'never')
+            )
+
+            return make_response(
+                jsonify({
+                    "success": True,
+                    "task_id": task.id
+                }), 
+                200
+            )
+
+        except Exception as err:
+            current_app.logger.error(
+                f"Error syncing connector source: {err}",
+                exc_info=True
+            )
+            return make_response(
+                jsonify({
+                    "success": False,
+                    "error": str(err)
+                }), 
+                400
+            )
+
+
+@connectors_ns.route("/api/connectors/callback-status")
+class ConnectorCallbackStatus(Resource):
+    @api.doc(description="Return HTML page with connector authentication status")
+    def get(self):
+        """Return HTML page with connector authentication status"""
+        try:
+            status = request.args.get('status', 'error')
+            message = request.args.get('message', '')
+            provider = request.args.get('provider', 'connector')
+            session_token = request.args.get('session_token', '')
+            user_email = request.args.get('user_email', '')
+            
+            html_content = f"""
+            <!DOCTYPE html>
+            <html>
+            <head>
+                <title>{provider.replace('_', ' ').title()} Authentication</title>
+                <style>
+                    body {{ font-family: Arial, sans-serif; text-align: center; padding: 40px; }}
+                    .container {{ max-width: 600px; margin: 0 auto; }}
+                    .success {{ color: #4CAF50; }}
+                    .error {{ color: #F44336; }}
+                    .cancelled {{ color: #FF9800; }}
+                </style>
+                <script>
+                    window.onload = function() {{
+                        const status = "{status}";
+                        const sessionToken = "{session_token}";
+                        const userEmail = "{user_email}";
+
+                        if (status === "success" && window.opener) {{
+                            window.opener.postMessage({{
+                                type: '{provider}_auth_success',
+                                session_token: sessionToken,
+                                user_email: userEmail
+                            }}, '*');
+
+                            setTimeout(() => window.close(), 3000);
+                        }} else if (status === "cancelled" || status === "error") {{
+                            setTimeout(() => window.close(), 3000);
+                        }}
+                    }};
+                </script>
+            </head>
+            <body>
+                <div class="container">
+                    <h2>{provider.replace('_', ' ').title()} Authentication</h2>
+                    <div class="{status}">
+                        <p>{message}</p>
+                        {f'<p>Connected as: {user_email}</p>' if status == 'success' else ''}
+                    </div>
+                    <p><small>You can close this window. {f"Your {provider.replace('_', ' ').title()} is now connected and ready to use." if status == 'success' else "Feel free to close this window."}</small></p>
+                </div>
+            </body>
+            </html>
+            """
+            
+            return make_response(html_content, 200, {'Content-Type': 'text/html'})
+        except Exception as e:
+            current_app.logger.error(f"Error rendering callback status page: {e}")
+            return make_response("Authentication error occurred", 500, {'Content-Type': 'text/html'})
+
+

application/api/internal/routes.py

@@ -0,0 +1,127 @@
+import os
+import datetime
+import json
+from flask import Blueprint, request, send_from_directory
+from werkzeug.utils import secure_filename
+from bson.objectid import ObjectId
+import logging
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.storage.storage_creator import StorageCreator
+
+
+logger = logging.getLogger(__name__)
+mongo = MongoDB.get_client()
+db = mongo[settings.MONGO_DB_NAME]
+conversations_collection = db["conversations"]
+sources_collection = db["sources"]
+
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+
+internal = Blueprint("internal", __name__)
+
+
+@internal.route("/api/download", methods=["get"])
+def download_file():
+    user = secure_filename(request.args.get("user"))
+    job_name = secure_filename(request.args.get("name"))
+    filename = secure_filename(request.args.get("file"))
+    save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name)
+    return send_from_directory(save_dir, filename, as_attachment=True)
+
+
+@internal.route("/api/upload_index", methods=["POST"])
+def upload_index_files():
+    """Upload two files(index.faiss, index.pkl) to the user's folder."""
+    if "user" not in request.form:
+        return {"status": "no user"}
+    user = request.form["user"] 
+    if "name" not in request.form:
+        return {"status": "no name"}
+    job_name = request.form["name"]
+    tokens = request.form["tokens"]
+    retriever = request.form["retriever"]
+    id = request.form["id"]
+    type = request.form["type"]
+    remote_data = request.form["remote_data"] if "remote_data" in request.form else None
+    sync_frequency = request.form["sync_frequency"] if "sync_frequency" in request.form else None
+    
+    file_path = request.form.get("file_path")
+    directory_structure = request.form.get("directory_structure")
+    
+    if directory_structure:
+        try:
+            directory_structure = json.loads(directory_structure)
+        except Exception:
+            logger.error("Error parsing directory_structure")
+            directory_structure = {}
+    else:
+        directory_structure = {}
+
+    storage = StorageCreator.get_storage()
+    index_base_path = f"indexes/{id}"
+    
+    if settings.VECTOR_STORE == "faiss":
+        if "file_faiss" not in request.files:
+            logger.error("No file_faiss part")
+            return {"status": "no file"}
+        file_faiss = request.files["file_faiss"]
+        if file_faiss.filename == "":
+            return {"status": "no file name"}
+        if "file_pkl" not in request.files:
+            logger.error("No file_pkl part")
+            return {"status": "no file"}
+        file_pkl = request.files["file_pkl"]
+        if file_pkl.filename == "":
+            return {"status": "no file name"}
+
+        # Save index files to storage
+        faiss_storage_path = f"{index_base_path}/index.faiss"
+        pkl_storage_path = f"{index_base_path}/index.pkl"
+        storage.save_file(file_faiss, faiss_storage_path)
+        storage.save_file(file_pkl, pkl_storage_path)
+
+
+    existing_entry = sources_collection.find_one({"_id": ObjectId(id)})
+    if existing_entry:
+        sources_collection.update_one(
+            {"_id": ObjectId(id)},
+            {
+                "$set": {
+                    "user": user,
+                    "name": job_name,
+                    "language": job_name,
+                    "date": datetime.datetime.now(),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "type": type,
+                    "tokens": tokens,
+                    "retriever": retriever,
+                    "remote_data": remote_data,
+                    "sync_frequency": sync_frequency,
+                    "file_path": file_path,
+                    "directory_structure": directory_structure,
+                }
+            },
+        )
+    else:
+        sources_collection.insert_one(
+            {
+                "_id": ObjectId(id),
+                "user": user,
+                "name": job_name,
+                "language": job_name,
+                "date": datetime.datetime.now(),
+                "model": settings.EMBEDDINGS_NAME,
+                "type": type,
+                "tokens": tokens,
+                "retriever": retriever,
+                "remote_data": remote_data,
+                "sync_frequency": sync_frequency,
+                "file_path": file_path,
+                "directory_structure": directory_structure,
+            }
+        )
+    return {"status": "ok"}

application/api/user/__init__.py

@@ -0,0 +1,5 @@
+"""User API module - provides all user-related API endpoints"""
+
+from .routes import user
+
+__all__ = ["user"]

application/api/user/agents/__init__.py

@@ -0,0 +1,7 @@
+"""Agents module."""
+
+from .routes import agents_ns
+from .sharing import agents_sharing_ns
+from .webhooks import agents_webhooks_ns
+
+__all__ = ["agents_ns", "agents_sharing_ns", "agents_webhooks_ns"]

application/api/user/agents/sharing.py

@@ -0,0 +1,263 @@
+"""Agent management sharing functionality."""
+
+import datetime
+import secrets
+
+from bson import DBRef
+from bson.objectid import ObjectId
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from application.api import api
+from application.core.settings import settings
+from application.api.user.base import (
+    agents_collection,
+    db,
+    ensure_user_doc,
+    resolve_tool_details,
+    user_tools_collection,
+    users_collection,
+)
+from application.utils import generate_image_url
+
+agents_sharing_ns = Namespace(
+    "agents", description="Agent management operations", path="/api"
+)
+
+
+@agents_sharing_ns.route("/shared_agent")
+class SharedAgent(Resource):
+    @api.doc(
+        params={
+            "token": "Shared token of the agent",
+        },
+        description="Get a shared agent by token or ID",
+    )
+    def get(self):
+        shared_token = request.args.get("token")
+
+        if not shared_token:
+            return make_response(
+                jsonify({"success": False, "message": "Token or ID is required"}), 400
+            )
+        try:
+            query = {
+                "shared_publicly": True,
+                "shared_token": shared_token,
+            }
+            shared_agent = agents_collection.find_one(query)
+            if not shared_agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Shared agent not found"}),
+                    404,
+                )
+            agent_id = str(shared_agent["_id"])
+            data = {
+                "id": agent_id,
+                "user": shared_agent.get("user", ""),
+                "name": shared_agent.get("name", ""),
+                "image": (
+                    generate_image_url(shared_agent["image"])
+                    if shared_agent.get("image")
+                    else ""
+                ),
+                "description": shared_agent.get("description", ""),
+                "source": (
+                    str(source_doc["_id"])
+                    if isinstance(shared_agent.get("source"), DBRef)
+                    and (source_doc := db.dereference(shared_agent.get("source")))
+                    else ""
+                ),
+                "chunks": shared_agent.get("chunks", "0"),
+                "retriever": shared_agent.get("retriever", "classic"),
+                "prompt_id": shared_agent.get("prompt_id", "default"),
+                "tools": shared_agent.get("tools", []),
+                "tool_details": resolve_tool_details(shared_agent.get("tools", [])),
+                "agent_type": shared_agent.get("agent_type", ""),
+                "status": shared_agent.get("status", ""),
+                "json_schema": shared_agent.get("json_schema"),
+                "limited_token_mode": shared_agent.get("limited_token_mode", False),
+                "token_limit": shared_agent.get("token_limit", settings.DEFAULT_AGENT_LIMITS["token_limit"]),
+                "limited_request_mode": shared_agent.get("limited_request_mode", False),
+                "request_limit": shared_agent.get("request_limit", settings.DEFAULT_AGENT_LIMITS["request_limit"]),
+                "created_at": shared_agent.get("createdAt", ""),
+                "updated_at": shared_agent.get("updatedAt", ""),
+                "shared": shared_agent.get("shared_publicly", False),
+                "shared_token": shared_agent.get("shared_token", ""),
+                "shared_metadata": shared_agent.get("shared_metadata", {}),
+            }
+
+            if data["tools"]:
+                enriched_tools = []
+                for tool in data["tools"]:
+                    tool_data = user_tools_collection.find_one({"_id": ObjectId(tool)})
+                    if tool_data:
+                        enriched_tools.append(tool_data.get("name", ""))
+                data["tools"] = enriched_tools
+            decoded_token = getattr(request, "decoded_token", None)
+            if decoded_token:
+                user_id = decoded_token.get("sub")
+                owner_id = shared_agent.get("user")
+
+                if user_id != owner_id:
+                    ensure_user_doc(user_id)
+                    users_collection.update_one(
+                        {"user_id": user_id},
+                        {"$addToSet": {"agent_preferences.shared_with_me": agent_id}},
+                    )
+            return make_response(jsonify(data), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving shared agent: {err}")
+            return make_response(jsonify({"success": False}), 400)
+
+
+@agents_sharing_ns.route("/shared_agents")
+class SharedAgents(Resource):
+    @api.doc(description="Get shared agents explicitly shared with the user")
+    def get(self):
+        try:
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False}), 401)
+            user_id = decoded_token.get("sub")
+
+            user_doc = ensure_user_doc(user_id)
+            shared_with_ids = user_doc.get("agent_preferences", {}).get(
+                "shared_with_me", []
+            )
+            shared_object_ids = [ObjectId(id) for id in shared_with_ids]
+
+            shared_agents_cursor = agents_collection.find(
+                {"_id": {"$in": shared_object_ids}, "shared_publicly": True}
+            )
+            shared_agents = list(shared_agents_cursor)
+
+            found_ids_set = {str(agent["_id"]) for agent in shared_agents}
+            stale_ids = [id for id in shared_with_ids if id not in found_ids_set]
+            if stale_ids:
+                users_collection.update_one(
+                    {"user_id": user_id},
+                    {"$pullAll": {"agent_preferences.shared_with_me": stale_ids}},
+                )
+            pinned_ids = set(user_doc.get("agent_preferences", {}).get("pinned", []))
+
+            list_shared_agents = [
+                {
+                    "id": str(agent["_id"]),
+                    "name": agent.get("name", ""),
+                    "description": agent.get("description", ""),
+                    "image": (
+                        generate_image_url(agent["image"]) if agent.get("image") else ""
+                    ),
+                    "tools": agent.get("tools", []),
+                    "tool_details": resolve_tool_details(agent.get("tools", [])),
+                    "agent_type": agent.get("agent_type", ""),
+                    "status": agent.get("status", ""),
+                    "json_schema": agent.get("json_schema"),
+                    "limited_token_mode": agent.get("limited_token_mode", False),
+                    "token_limit": agent.get("token_limit", settings.DEFAULT_AGENT_LIMITS["token_limit"]),
+                    "limited_request_mode": agent.get("limited_request_mode", False),
+                    "request_limit": agent.get("request_limit", settings.DEFAULT_AGENT_LIMITS["request_limit"]),
+                    "created_at": agent.get("createdAt", ""),
+                    "updated_at": agent.get("updatedAt", ""),
+                    "pinned": str(agent["_id"]) in pinned_ids,
+                    "shared": agent.get("shared_publicly", False),
+                    "shared_token": agent.get("shared_token", ""),
+                    "shared_metadata": agent.get("shared_metadata", {}),
+                }
+                for agent in shared_agents
+            ]
+
+            return make_response(jsonify(list_shared_agents), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving shared agents: {err}")
+            return make_response(jsonify({"success": False}), 400)
+
+
+@agents_sharing_ns.route("/share_agent")
+class ShareAgent(Resource):
+    @api.expect(
+        api.model(
+            "ShareAgentModel",
+            {
+                "id": fields.String(required=True, description="ID of the agent"),
+                "shared": fields.Boolean(
+                    required=True, description="Share or unshare the agent"
+                ),
+                "username": fields.String(
+                    required=False, description="Name of the user"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Share or unshare an agent")
+    def put(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+
+        data = request.get_json()
+        if not data:
+            return make_response(
+                jsonify({"success": False, "message": "Missing JSON body"}), 400
+            )
+        agent_id = data.get("id")
+        shared = data.get("shared")
+        username = data.get("username", "")
+
+        if not agent_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        if shared is None:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Shared parameter is required and must be true or false",
+                    }
+                ),
+                400,
+            )
+        try:
+            try:
+                agent_oid = ObjectId(agent_id)
+            except Exception:
+                return make_response(
+                    jsonify({"success": False, "message": "Invalid agent ID"}), 400
+                )
+            agent = agents_collection.find_one({"_id": agent_oid, "user": user})
+            if not agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Agent not found"}), 404
+                )
+            if shared:
+                shared_metadata = {
+                    "shared_by": username,
+                    "shared_at": datetime.datetime.now(datetime.timezone.utc),
+                }
+                shared_token = secrets.token_urlsafe(32)
+                agents_collection.update_one(
+                    {"_id": agent_oid, "user": user},
+                    {
+                        "$set": {
+                            "shared_publicly": shared,
+                            "shared_metadata": shared_metadata,
+                            "shared_token": shared_token,
+                        }
+                    },
+                )
+            else:
+                agents_collection.update_one(
+                    {"_id": agent_oid, "user": user},
+                    {"$set": {"shared_publicly": shared, "shared_token": None}},
+                    {"$unset": {"shared_metadata": ""}},
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error sharing/unsharing agent: {err}")
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+        shared_token = shared_token if shared else None
+        return make_response(
+            jsonify({"success": True, "shared_token": shared_token}), 200
+        )

application/api/user/agents/webhooks.py

@@ -0,0 +1,119 @@
+"""Agent management webhook handlers."""
+
+import secrets
+
+from bson.objectid import ObjectId
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+from application.api import api
+from application.api.user.base import agents_collection, require_agent
+from application.api.user.tasks import process_agent_webhook
+from application.core.settings import settings
+
+
+agents_webhooks_ns = Namespace(
+    "agents", description="Agent management operations", path="/api"
+)
+
+
+@agents_webhooks_ns.route("/agent_webhook")
+class AgentWebhook(Resource):
+    @api.doc(
+        params={"id": "ID of the agent"},
+        description="Generate webhook URL for the agent",
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        agent_id = request.args.get("id")
+        if not agent_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        try:
+            agent = agents_collection.find_one(
+                {"_id": ObjectId(agent_id), "user": user}
+            )
+            if not agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Agent not found"}), 404
+                )
+            webhook_token = agent.get("incoming_webhook_token")
+            if not webhook_token:
+                webhook_token = secrets.token_urlsafe(32)
+                agents_collection.update_one(
+                    {"_id": ObjectId(agent_id), "user": user},
+                    {"$set": {"incoming_webhook_token": webhook_token}},
+                )
+            base_url = settings.API_URL.rstrip("/")
+            full_webhook_url = f"{base_url}/api/webhooks/agents/{webhook_token}"
+        except Exception as err:
+            current_app.logger.error(
+                f"Error generating webhook URL: {err}", exc_info=True
+            )
+            return make_response(
+                jsonify({"success": False, "message": "Error generating webhook URL"}),
+                400,
+            )
+        return make_response(
+            jsonify({"success": True, "webhook_url": full_webhook_url}), 200
+        )
+
+
+@agents_webhooks_ns.route("/webhooks/agents/<string:webhook_token>")
+class AgentWebhookListener(Resource):
+    method_decorators = [require_agent]
+
+    def _enqueue_webhook_task(self, agent_id_str, payload, source_method):
+        if not payload:
+            current_app.logger.warning(
+                f"Webhook ({source_method}) received for agent {agent_id_str} with empty payload."
+            )
+        current_app.logger.info(
+            f"Incoming {source_method} webhook for agent {agent_id_str}. Enqueuing task with payload: {payload}"
+        )
+
+        try:
+            task = process_agent_webhook.delay(
+                agent_id=agent_id_str,
+                payload=payload,
+            )
+            current_app.logger.info(
+                f"Task {task.id} enqueued for agent {agent_id_str} ({source_method})."
+            )
+            return make_response(jsonify({"success": True, "task_id": task.id}), 200)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error enqueuing webhook task ({source_method}) for agent {agent_id_str}: {err}",
+                exc_info=True,
+            )
+            return make_response(
+                jsonify({"success": False, "message": "Error processing webhook"}), 500
+            )
+
+    @api.doc(
+        description="Webhook listener for agent events (POST). Expects JSON payload, which is used to trigger processing.",
+    )
+    def post(self, webhook_token, agent, agent_id_str):
+        payload = request.get_json()
+        if payload is None:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Invalid or missing JSON data in request body",
+                    }
+                ),
+                400,
+            )
+        return self._enqueue_webhook_task(agent_id_str, payload, source_method="POST")
+
+    @api.doc(
+        description="Webhook listener for agent events (GET). Uses URL query parameters as payload to trigger processing.",
+    )
+    def get(self, webhook_token, agent, agent_id_str):
+        payload = request.args.to_dict(flat=True)
+        return self._enqueue_webhook_task(agent_id_str, payload, source_method="GET")

application/api/user/analytics/__init__.py

@@ -0,0 +1,5 @@
+"""Analytics module."""
+
+from .routes import analytics_ns
+
+__all__ = ["analytics_ns"]

application/api/user/analytics/routes.py

@@ -0,0 +1,540 @@
+"""Analytics and reporting routes."""
+
+import datetime
+
+from bson.objectid import ObjectId
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from application.api import api
+from application.api.user.base import (
+    agents_collection,
+    conversations_collection,
+    generate_date_range,
+    generate_hourly_range,
+    generate_minute_range,
+    token_usage_collection,
+    user_logs_collection,
+)
+
+analytics_ns = Namespace(
+    "analytics", description="Analytics and reporting operations", path="/api"
+)
+
+
+@analytics_ns.route("/get_message_analytics")
+class GetMessageAnalytics(Resource):
+    get_message_analytics_model = api.model(
+        "GetMessageAnalyticsModel",
+        {
+            "api_key_id": fields.String(required=False, description="API Key ID"),
+            "filter_option": fields.String(
+                required=False,
+                description="Filter option for analytics",
+                default="last_30_days",
+                enum=[
+                    "last_hour",
+                    "last_24_hour",
+                    "last_7_days",
+                    "last_15_days",
+                    "last_30_days",
+                ],
+            ),
+        },
+    )
+
+    @api.expect(get_message_analytics_model)
+    @api.doc(description="Get message analytics based on filter option")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        api_key_id = data.get("api_key_id")
+        filter_option = data.get("filter_option", "last_30_days")
+
+        try:
+            api_key = (
+                agents_collection.find_one({"_id": ObjectId(api_key_id), "user": user})[
+                    "key"
+                ]
+                if api_key_id
+                else None
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error getting API key: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        end_date = datetime.datetime.now(datetime.timezone.utc)
+
+        if filter_option == "last_hour":
+            start_date = end_date - datetime.timedelta(hours=1)
+            group_format = "%Y-%m-%d %H:%M:00"
+        elif filter_option == "last_24_hour":
+            start_date = end_date - datetime.timedelta(hours=24)
+            group_format = "%Y-%m-%d %H:00"
+        else:
+            if filter_option in ["last_7_days", "last_15_days", "last_30_days"]:
+                filter_days = (
+                    6
+                    if filter_option == "last_7_days"
+                    else 14 if filter_option == "last_15_days" else 29
+                )
+            else:
+                return make_response(
+                    jsonify({"success": False, "message": "Invalid option"}), 400
+                )
+            start_date = end_date - datetime.timedelta(days=filter_days)
+            start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+            end_date = end_date.replace(
+                hour=23, minute=59, second=59, microsecond=999999
+            )
+            group_format = "%Y-%m-%d"
+        try:
+            match_stage = {
+                "$match": {
+                    "user": user,
+                }
+            }
+            if api_key:
+                match_stage["$match"]["api_key"] = api_key
+            pipeline = [
+                match_stage,
+                {"$unwind": "$queries"},
+                {
+                    "$match": {
+                        "queries.timestamp": {"$gte": start_date, "$lte": end_date}
+                    }
+                },
+                {
+                    "$group": {
+                        "_id": {
+                            "$dateToString": {
+                                "format": group_format,
+                                "date": "$queries.timestamp",
+                            }
+                        },
+                        "count": {"$sum": 1},
+                    }
+                },
+                {"$sort": {"_id": 1}},
+            ]
+
+            message_data = conversations_collection.aggregate(pipeline)
+
+            if filter_option == "last_hour":
+                intervals = generate_minute_range(start_date, end_date)
+            elif filter_option == "last_24_hour":
+                intervals = generate_hourly_range(start_date, end_date)
+            else:
+                intervals = generate_date_range(start_date, end_date)
+            daily_messages = {interval: 0 for interval in intervals}
+
+            for entry in message_data:
+                daily_messages[entry["_id"]] = entry["count"]
+        except Exception as err:
+            current_app.logger.error(
+                f"Error getting message analytics: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(
+            jsonify({"success": True, "messages": daily_messages}), 200
+        )
+
+
+@analytics_ns.route("/get_token_analytics")
+class GetTokenAnalytics(Resource):
+    get_token_analytics_model = api.model(
+        "GetTokenAnalyticsModel",
+        {
+            "api_key_id": fields.String(required=False, description="API Key ID"),
+            "filter_option": fields.String(
+                required=False,
+                description="Filter option for analytics",
+                default="last_30_days",
+                enum=[
+                    "last_hour",
+                    "last_24_hour",
+                    "last_7_days",
+                    "last_15_days",
+                    "last_30_days",
+                ],
+            ),
+        },
+    )
+
+    @api.expect(get_token_analytics_model)
+    @api.doc(description="Get token analytics data")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        api_key_id = data.get("api_key_id")
+        filter_option = data.get("filter_option", "last_30_days")
+
+        try:
+            api_key = (
+                agents_collection.find_one({"_id": ObjectId(api_key_id), "user": user})[
+                    "key"
+                ]
+                if api_key_id
+                else None
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error getting API key: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        end_date = datetime.datetime.now(datetime.timezone.utc)
+
+        if filter_option == "last_hour":
+            start_date = end_date - datetime.timedelta(hours=1)
+            group_format = "%Y-%m-%d %H:%M:00"
+            group_stage = {
+                "$group": {
+                    "_id": {
+                        "minute": {
+                            "$dateToString": {
+                                "format": group_format,
+                                "date": "$timestamp",
+                            }
+                        }
+                    },
+                    "total_tokens": {
+                        "$sum": {"$add": ["$prompt_tokens", "$generated_tokens"]}
+                    },
+                }
+            }
+        elif filter_option == "last_24_hour":
+            start_date = end_date - datetime.timedelta(hours=24)
+            group_format = "%Y-%m-%d %H:00"
+            group_stage = {
+                "$group": {
+                    "_id": {
+                        "hour": {
+                            "$dateToString": {
+                                "format": group_format,
+                                "date": "$timestamp",
+                            }
+                        }
+                    },
+                    "total_tokens": {
+                        "$sum": {"$add": ["$prompt_tokens", "$generated_tokens"]}
+                    },
+                }
+            }
+        else:
+            if filter_option in ["last_7_days", "last_15_days", "last_30_days"]:
+                filter_days = (
+                    6
+                    if filter_option == "last_7_days"
+                    else (14 if filter_option == "last_15_days" else 29)
+                )
+            else:
+                return make_response(
+                    jsonify({"success": False, "message": "Invalid option"}), 400
+                )
+            start_date = end_date - datetime.timedelta(days=filter_days)
+            start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+            end_date = end_date.replace(
+                hour=23, minute=59, second=59, microsecond=999999
+            )
+            group_format = "%Y-%m-%d"
+            group_stage = {
+                "$group": {
+                    "_id": {
+                        "day": {
+                            "$dateToString": {
+                                "format": group_format,
+                                "date": "$timestamp",
+                            }
+                        }
+                    },
+                    "total_tokens": {
+                        "$sum": {"$add": ["$prompt_tokens", "$generated_tokens"]}
+                    },
+                }
+            }
+        try:
+            match_stage = {
+                "$match": {
+                    "user_id": user,
+                    "timestamp": {"$gte": start_date, "$lte": end_date},
+                }
+            }
+            if api_key:
+                match_stage["$match"]["api_key"] = api_key
+            token_usage_data = token_usage_collection.aggregate(
+                [
+                    match_stage,
+                    group_stage,
+                    {"$sort": {"_id": 1}},
+                ]
+            )
+
+            if filter_option == "last_hour":
+                intervals = generate_minute_range(start_date, end_date)
+            elif filter_option == "last_24_hour":
+                intervals = generate_hourly_range(start_date, end_date)
+            else:
+                intervals = generate_date_range(start_date, end_date)
+            daily_token_usage = {interval: 0 for interval in intervals}
+
+            for entry in token_usage_data:
+                if filter_option == "last_hour":
+                    daily_token_usage[entry["_id"]["minute"]] = entry["total_tokens"]
+                elif filter_option == "last_24_hour":
+                    daily_token_usage[entry["_id"]["hour"]] = entry["total_tokens"]
+                else:
+                    daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"]
+        except Exception as err:
+            current_app.logger.error(
+                f"Error getting token analytics: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(
+            jsonify({"success": True, "token_usage": daily_token_usage}), 200
+        )
+
+
+@analytics_ns.route("/get_feedback_analytics")
+class GetFeedbackAnalytics(Resource):
+    get_feedback_analytics_model = api.model(
+        "GetFeedbackAnalyticsModel",
+        {
+            "api_key_id": fields.String(required=False, description="API Key ID"),
+            "filter_option": fields.String(
+                required=False,
+                description="Filter option for analytics",
+                default="last_30_days",
+                enum=[
+                    "last_hour",
+                    "last_24_hour",
+                    "last_7_days",
+                    "last_15_days",
+                    "last_30_days",
+                ],
+            ),
+        },
+    )
+
+    @api.expect(get_feedback_analytics_model)
+    @api.doc(description="Get feedback analytics data")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        api_key_id = data.get("api_key_id")
+        filter_option = data.get("filter_option", "last_30_days")
+
+        try:
+            api_key = (
+                agents_collection.find_one({"_id": ObjectId(api_key_id), "user": user})[
+                    "key"
+                ]
+                if api_key_id
+                else None
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error getting API key: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        end_date = datetime.datetime.now(datetime.timezone.utc)
+
+        if filter_option == "last_hour":
+            start_date = end_date - datetime.timedelta(hours=1)
+            group_format = "%Y-%m-%d %H:%M:00"
+            date_field = {
+                "$dateToString": {
+                    "format": group_format,
+                    "date": "$queries.feedback_timestamp",
+                }
+            }
+        elif filter_option == "last_24_hour":
+            start_date = end_date - datetime.timedelta(hours=24)
+            group_format = "%Y-%m-%d %H:00"
+            date_field = {
+                "$dateToString": {
+                    "format": group_format,
+                    "date": "$queries.feedback_timestamp",
+                }
+            }
+        else:
+            if filter_option in ["last_7_days", "last_15_days", "last_30_days"]:
+                filter_days = (
+                    6
+                    if filter_option == "last_7_days"
+                    else (14 if filter_option == "last_15_days" else 29)
+                )
+            else:
+                return make_response(
+                    jsonify({"success": False, "message": "Invalid option"}), 400
+                )
+            start_date = end_date - datetime.timedelta(days=filter_days)
+            start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+            end_date = end_date.replace(
+                hour=23, minute=59, second=59, microsecond=999999
+            )
+            group_format = "%Y-%m-%d"
+            date_field = {
+                "$dateToString": {
+                    "format": group_format,
+                    "date": "$queries.feedback_timestamp",
+                }
+            }
+        try:
+            match_stage = {
+                "$match": {
+                    "queries.feedback_timestamp": {
+                        "$gte": start_date,
+                        "$lte": end_date,
+                    },
+                    "queries.feedback": {"$exists": True},
+                }
+            }
+            if api_key:
+                match_stage["$match"]["api_key"] = api_key
+            pipeline = [
+                match_stage,
+                {"$unwind": "$queries"},
+                {"$match": {"queries.feedback": {"$exists": True}}},
+                {
+                    "$group": {
+                        "_id": {"time": date_field, "feedback": "$queries.feedback"},
+                        "count": {"$sum": 1},
+                    }
+                },
+                {
+                    "$group": {
+                        "_id": "$_id.time",
+                        "positive": {
+                            "$sum": {
+                                "$cond": [
+                                    {"$eq": ["$_id.feedback", "LIKE"]},
+                                    "$count",
+                                    0,
+                                ]
+                            }
+                        },
+                        "negative": {
+                            "$sum": {
+                                "$cond": [
+                                    {"$eq": ["$_id.feedback", "DISLIKE"]},
+                                    "$count",
+                                    0,
+                                ]
+                            }
+                        },
+                    }
+                },
+                {"$sort": {"_id": 1}},
+            ]
+
+            feedback_data = conversations_collection.aggregate(pipeline)
+
+            if filter_option == "last_hour":
+                intervals = generate_minute_range(start_date, end_date)
+            elif filter_option == "last_24_hour":
+                intervals = generate_hourly_range(start_date, end_date)
+            else:
+                intervals = generate_date_range(start_date, end_date)
+            daily_feedback = {
+                interval: {"positive": 0, "negative": 0} for interval in intervals
+            }
+
+            for entry in feedback_data:
+                daily_feedback[entry["_id"]] = {
+                    "positive": entry["positive"],
+                    "negative": entry["negative"],
+                }
+        except Exception as err:
+            current_app.logger.error(
+                f"Error getting feedback analytics: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(
+            jsonify({"success": True, "feedback": daily_feedback}), 200
+        )
+
+
+@analytics_ns.route("/get_user_logs")
+class GetUserLogs(Resource):
+    get_user_logs_model = api.model(
+        "GetUserLogsModel",
+        {
+            "page": fields.Integer(
+                required=False,
+                description="Page number for pagination",
+                default=1,
+            ),
+            "api_key_id": fields.String(required=False, description="API Key ID"),
+            "page_size": fields.Integer(
+                required=False,
+                description="Number of logs per page",
+                default=10,
+            ),
+        },
+    )
+
+    @api.expect(get_user_logs_model)
+    @api.doc(description="Get user logs with pagination")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        page = int(data.get("page", 1))
+        api_key_id = data.get("api_key_id")
+        page_size = int(data.get("page_size", 10))
+        skip = (page - 1) * page_size
+
+        try:
+            api_key = (
+                agents_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
+                if api_key_id
+                else None
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error getting API key: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        query = {"user": user}
+        if api_key:
+            query = {"api_key": api_key}
+        items_cursor = (
+            user_logs_collection.find(query)
+            .sort("timestamp", -1)
+            .skip(skip)
+            .limit(page_size + 1)
+        )
+        items = list(items_cursor)
+
+        results = [
+            {
+                "id": str(item.get("_id")),
+                "action": item.get("action"),
+                "level": item.get("level"),
+                "user": item.get("user"),
+                "question": item.get("question"),
+                "sources": item.get("sources"),
+                "retriever_params": item.get("retriever_params"),
+                "timestamp": item.get("timestamp"),
+            }
+            for item in items[:page_size]
+        ]
+
+        has_more = len(items) > page_size
+
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "logs": results,
+                    "page": page,
+                    "page_size": page_size,
+                    "has_more": has_more,
+                }
+            ),
+            200,
+        )

application/api/user/attachments/__init__.py

@@ -0,0 +1,5 @@
+"""Attachments module."""
+
+from .routes import attachments_ns
+
+__all__ = ["attachments_ns"]

application/api/user/attachments/routes.py

@@ -0,0 +1,198 @@
+"""File attachments and media routes."""
+
+import os
+
+from bson.objectid import ObjectId
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from application.api import api
+from application.api.user.base import agents_collection, storage
+from application.api.user.tasks import store_attachment
+from application.core.settings import settings
+from application.tts.tts_creator import TTSCreator
+from application.utils import safe_filename
+
+
+attachments_ns = Namespace(
+    "attachments", description="File attachments and media operations", path="/api"
+)
+
+
+@attachments_ns.route("/store_attachment")
+class StoreAttachment(Resource):
+    @api.expect(
+        api.model(
+            "AttachmentModel",
+            {
+                "file": fields.Raw(required=True, description="File(s) to upload"),
+                "api_key": fields.String(
+                    required=False, description="API key (optional)"
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Stores one or multiple attachments without vectorization or training. Supports user or API key authentication."
+    )
+    def post(self):
+        decoded_token = getattr(request, "decoded_token", None)
+        api_key = request.form.get("api_key") or request.args.get("api_key")
+        
+        files = request.files.getlist("file")
+        if not files:
+            single_file = request.files.get("file")
+            if single_file:
+                files = [single_file]
+        
+        if not files or all(f.filename == "" for f in files):
+            return make_response(
+                jsonify({"status": "error", "message": "Missing file(s)"}),
+                400,
+            )
+        
+        user = None
+        if decoded_token:
+            user = safe_filename(decoded_token.get("sub"))
+        elif api_key:
+            agent = agents_collection.find_one({"key": api_key})
+            if not agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Invalid API key"}), 401
+                )
+            user = safe_filename(agent.get("user"))
+        else:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}), 401
+            )
+        
+        try:
+            tasks = []
+            errors = []
+            original_file_count = len(files)
+            
+            for idx, file in enumerate(files):
+                try:
+                    attachment_id = ObjectId()
+                    original_filename = safe_filename(os.path.basename(file.filename))
+                    relative_path = f"{settings.UPLOAD_FOLDER}/{user}/attachments/{str(attachment_id)}/{original_filename}"
+
+                    metadata = storage.save_file(file, relative_path)
+                    file_info = {
+                        "filename": original_filename,
+                        "attachment_id": str(attachment_id),
+                        "path": relative_path,
+                        "metadata": metadata,
+                    }
+
+                    task = store_attachment.delay(file_info, user)
+                    tasks.append({
+                        "task_id": task.id,
+                        "filename": original_filename,
+                        "attachment_id": str(attachment_id),
+                    })
+                except Exception as file_err:
+                    current_app.logger.error(f"Error processing file {idx} ({file.filename}): {file_err}", exc_info=True)
+                    errors.append({
+                        "filename": file.filename,
+                        "error": str(file_err)
+                    })
+            
+            if not tasks:
+                error_msg = "No valid files to upload"
+                if errors:
+                    error_msg += f". Errors: {errors}"
+                return make_response(
+                    jsonify({"status": "error", "message": error_msg, "errors": errors}),
+                    400,
+                )
+            
+            if original_file_count == 1 and len(tasks) == 1:
+                current_app.logger.info("Returning single task_id response")
+                return make_response(
+                    jsonify(
+                        {
+                            "success": True,
+                            "task_id": tasks[0]["task_id"],
+                            "message": "File uploaded successfully. Processing started.",
+                        }
+                    ),
+                    200,
+                )
+            else:
+                response_data = {
+                    "success": True,
+                    "tasks": tasks,
+                    "message": f"{len(tasks)} file(s) uploaded successfully. Processing started.",
+                }
+                if errors:
+                    response_data["errors"] = errors
+                    response_data["message"] += f" {len(errors)} file(s) failed."
+                
+                return make_response(
+                    jsonify(response_data),
+                    200,
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error storing attachment: {err}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+
+@attachments_ns.route("/images/<path:image_path>")
+class ServeImage(Resource):
+    @api.doc(description="Serve an image from storage")
+    def get(self, image_path):
+        try:
+            file_obj = storage.get_file(image_path)
+            extension = image_path.split(".")[-1].lower()
+            content_type = f"image/{extension}"
+            if extension == "jpg":
+                content_type = "image/jpeg"
+            response = make_response(file_obj.read())
+            response.headers.set("Content-Type", content_type)
+            response.headers.set("Cache-Control", "max-age=86400")
+
+            return response
+        except FileNotFoundError:
+            return make_response(
+                jsonify({"success": False, "message": "Image not found"}), 404
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error serving image: {e}")
+            return make_response(
+                jsonify({"success": False, "message": "Error retrieving image"}), 500
+            )
+
+
+@attachments_ns.route("/tts")
+class TextToSpeech(Resource):
+    tts_model = api.model(
+        "TextToSpeechModel",
+        {
+            "text": fields.String(
+                required=True, description="Text to be synthesized as audio"
+            ),
+        },
+    )
+
+    @api.expect(tts_model)
+    @api.doc(description="Synthesize audio speech from text")
+    def post(self):
+        data = request.get_json()
+        text = data["text"]
+        try:
+            tts_instance = TTSCreator.create_tts(settings.TTS_PROVIDER)
+            audio_base64, detected_language = tts_instance.text_to_speech(text)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "audio_base64": audio_base64,
+                        "lang": detected_language,
+                    }
+                ),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error synthesizing audio: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)

application/api/user/base.py

@@ -0,0 +1,222 @@
+"""
+Shared utilities, database connections, and helper functions for user API routes.
+"""
+
+import datetime
+import os
+import uuid
+from functools import wraps
+from typing import Optional, Tuple
+
+from bson.objectid import ObjectId
+from flask import current_app, jsonify, make_response, Response
+from pymongo import ReturnDocument
+from werkzeug.utils import secure_filename
+
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+from application.storage.storage_creator import StorageCreator
+from application.vectorstore.vector_creator import VectorCreator
+
+
+storage = StorageCreator.get_storage()
+
+
+mongo = MongoDB.get_client()
+db = mongo[settings.MONGO_DB_NAME]
+
+
+conversations_collection = db["conversations"]
+sources_collection = db["sources"]
+prompts_collection = db["prompts"]
+feedback_collection = db["feedback"]
+agents_collection = db["agents"]
+token_usage_collection = db["token_usage"]
+shared_conversations_collections = db["shared_conversations"]
+users_collection = db["users"]
+user_logs_collection = db["user_logs"]
+user_tools_collection = db["user_tools"]
+attachments_collection = db["attachments"]
+
+
+try:
+    agents_collection.create_index(
+        [("shared", 1)],
+        name="shared_index",
+        background=True,
+    )
+    users_collection.create_index("user_id", unique=True)
+except Exception as e:
+    print("Error creating indexes:", e)
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+
+def generate_minute_range(start_date, end_date):
+    """Generate a dictionary with minute-level time ranges."""
+    return {
+        (start_date + datetime.timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:00"): 0
+        for i in range(int((end_date - start_date).total_seconds() // 60) + 1)
+    }
+
+
+def generate_hourly_range(start_date, end_date):
+    """Generate a dictionary with hourly time ranges."""
+    return {
+        (start_date + datetime.timedelta(hours=i)).strftime("%Y-%m-%d %H:00"): 0
+        for i in range(int((end_date - start_date).total_seconds() // 3600) + 1)
+    }
+
+
+def generate_date_range(start_date, end_date):
+    """Generate a dictionary with daily date ranges."""
+    return {
+        (start_date + datetime.timedelta(days=i)).strftime("%Y-%m-%d"): 0
+        for i in range((end_date - start_date).days + 1)
+    }
+
+
+def ensure_user_doc(user_id):
+    """
+    Ensure user document exists with proper agent preferences structure.
+
+    Args:
+        user_id: The user ID to ensure
+
+    Returns:
+        The user document
+    """
+    default_prefs = {
+        "pinned": [],
+        "shared_with_me": [],
+    }
+
+    user_doc = users_collection.find_one_and_update(
+        {"user_id": user_id},
+        {"$setOnInsert": {"agent_preferences": default_prefs}},
+        upsert=True,
+        return_document=ReturnDocument.AFTER,
+    )
+
+    prefs = user_doc.get("agent_preferences", {})
+    updates = {}
+    if "pinned" not in prefs:
+        updates["agent_preferences.pinned"] = []
+    if "shared_with_me" not in prefs:
+        updates["agent_preferences.shared_with_me"] = []
+    if updates:
+        users_collection.update_one({"user_id": user_id}, {"$set": updates})
+        user_doc = users_collection.find_one({"user_id": user_id})
+    return user_doc
+
+
+def resolve_tool_details(tool_ids):
+    """
+    Resolve tool IDs to their details.
+
+    Args:
+        tool_ids: List of tool IDs
+
+    Returns:
+        List of tool details with id, name, and display_name
+    """
+    tools = user_tools_collection.find(
+        {"_id": {"$in": [ObjectId(tid) for tid in tool_ids]}}
+    )
+    return [
+        {
+            "id": str(tool["_id"]),
+            "name": tool.get("name", ""),
+            "display_name": tool.get("displayName", tool.get("name", "")),
+        }
+        for tool in tools
+    ]
+
+
+def get_vector_store(source_id):
+    """
+    Get the Vector Store for a given source ID.
+
+    Args:
+        source_id (str): source id of the document
+
+    Returns:
+        Vector store instance
+    """
+    store = VectorCreator.create_vectorstore(
+        settings.VECTOR_STORE,
+        source_id=source_id,
+        embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+    )
+    return store
+
+
+def handle_image_upload(
+    request, existing_url: str, user: str, storage, base_path: str = "attachments/"
+) -> Tuple[str, Optional[Response]]:
+    """
+    Handle image file upload from request.
+
+    Args:
+        request: Flask request object
+        existing_url: Existing image URL (fallback)
+        user: User ID
+        storage: Storage instance
+        base_path: Base path for upload
+
+    Returns:
+        Tuple of (image_url, error_response)
+    """
+    image_url = existing_url
+
+    if "image" in request.files:
+        file = request.files["image"]
+        if file.filename != "":
+            filename = secure_filename(file.filename)
+            upload_path = f"{settings.UPLOAD_FOLDER.rstrip('/')}/{user}/{base_path.rstrip('/')}/{uuid.uuid4()}_{filename}"
+            try:
+                storage.save_file(file, upload_path, storage_class="STANDARD")
+                image_url = upload_path
+            except Exception as e:
+                current_app.logger.error(f"Error uploading image: {e}")
+                return None, make_response(
+                    jsonify({"success": False, "message": "Image upload failed"}),
+                    400,
+                )
+    return image_url, None
+
+
+def require_agent(func):
+    """
+    Decorator to require valid agent webhook token.
+
+    Args:
+        func: Function to decorate
+
+    Returns:
+        Wrapped function
+    """
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        webhook_token = kwargs.get("webhook_token")
+        if not webhook_token:
+            return make_response(
+                jsonify({"success": False, "message": "Webhook token missing"}), 400
+            )
+        agent = agents_collection.find_one(
+            {"incoming_webhook_token": webhook_token}, {"_id": 1}
+        )
+        if not agent:
+            current_app.logger.warning(
+                f"Webhook attempt with invalid token: {webhook_token}"
+            )
+            return make_response(
+                jsonify({"success": False, "message": "Agent not found"}), 404
+            )
+        kwargs["agent"] = agent
+        kwargs["agent_id_str"] = str(agent["_id"])
+        return func(*args, **kwargs)
+
+    return wrapper

application/api/user/conversations/__init__.py

@@ -0,0 +1,5 @@
+"""Conversation management module."""
+
+from .routes import conversations_ns
+
+__all__ = ["conversations_ns"]

application/api/user/conversations/routes.py

@@ -0,0 +1,280 @@
+"""Conversation management routes."""
+
+import datetime
+
+from bson.objectid import ObjectId
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from application.api import api
+from application.api.user.base import attachments_collection, conversations_collection
+from application.utils import check_required_fields
+
+conversations_ns = Namespace(
+    "conversations", description="Conversation management operations", path="/api"
+)
+
+
+@conversations_ns.route("/delete_conversation")
+class DeleteConversation(Resource):
+    @api.doc(
+        description="Deletes a conversation by ID",
+        params={"id": "The ID of the conversation to delete"},
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        conversation_id = request.args.get("id")
+        if not conversation_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        try:
+            conversations_collection.delete_one(
+                {"_id": ObjectId(conversation_id), "user": decoded_token["sub"]}
+            )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error deleting conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/delete_all_conversations")
+class DeleteAllConversations(Resource):
+    @api.doc(
+        description="Deletes all conversations for a specific user",
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user_id = decoded_token.get("sub")
+        try:
+            conversations_collection.delete_many({"user": user_id})
+        except Exception as err:
+            current_app.logger.error(
+                f"Error deleting all conversations: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/get_conversations")
+class GetConversations(Resource):
+    @api.doc(
+        description="Retrieve a list of the latest 30 conversations (excluding API key conversations)",
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        try:
+            conversations = (
+                conversations_collection.find(
+                    {
+                        "$or": [
+                            {"api_key": {"$exists": False}},
+                            {"agent_id": {"$exists": True}},
+                        ],
+                        "user": decoded_token.get("sub"),
+                    }
+                )
+                .sort("date", -1)
+                .limit(30)
+            )
+
+            list_conversations = [
+                {
+                    "id": str(conversation["_id"]),
+                    "name": conversation["name"],
+                    "agent_id": conversation.get("agent_id", None),
+                    "is_shared_usage": conversation.get("is_shared_usage", False),
+                    "shared_token": conversation.get("shared_token", None),
+                }
+                for conversation in conversations
+            ]
+        except Exception as err:
+            current_app.logger.error(
+                f"Error retrieving conversations: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify(list_conversations), 200)
+
+
+@conversations_ns.route("/get_single_conversation")
+class GetSingleConversation(Resource):
+    @api.doc(
+        description="Retrieve a single conversation by ID",
+        params={"id": "The conversation ID"},
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        conversation_id = request.args.get("id")
+        if not conversation_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        try:
+            conversation = conversations_collection.find_one(
+                {"_id": ObjectId(conversation_id), "user": decoded_token.get("sub")}
+            )
+            if not conversation:
+                return make_response(jsonify({"status": "not found"}), 404)
+            # Process queries to include attachment names
+
+            queries = conversation["queries"]
+            for query in queries:
+                if "attachments" in query and query["attachments"]:
+                    attachment_details = []
+                    for attachment_id in query["attachments"]:
+                        try:
+                            attachment = attachments_collection.find_one(
+                                {"_id": ObjectId(attachment_id)}
+                            )
+                            if attachment:
+                                attachment_details.append(
+                                    {
+                                        "id": str(attachment["_id"]),
+                                        "fileName": attachment.get(
+                                            "filename", "Unknown file"
+                                        ),
+                                    }
+                                )
+                        except Exception as e:
+                            current_app.logger.error(
+                                f"Error retrieving attachment {attachment_id}: {e}",
+                                exc_info=True,
+                            )
+                    query["attachments"] = attachment_details
+        except Exception as err:
+            current_app.logger.error(
+                f"Error retrieving conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        data = {
+            "queries": queries,
+            "agent_id": conversation.get("agent_id"),
+            "is_shared_usage": conversation.get("is_shared_usage", False),
+            "shared_token": conversation.get("shared_token", None),
+        }
+        return make_response(jsonify(data), 200)
+
+
+@conversations_ns.route("/update_conversation_name")
+class UpdateConversationName(Resource):
+    @api.expect(
+        api.model(
+            "UpdateConversationModel",
+            {
+                "id": fields.String(required=True, description="Conversation ID"),
+                "name": fields.String(
+                    required=True, description="New name of the conversation"
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Updates the name of a conversation",
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        data = request.get_json()
+        required_fields = ["id", "name"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            conversations_collection.update_one(
+                {"_id": ObjectId(data["id"]), "user": decoded_token.get("sub")},
+                {"$set": {"name": data["name"]}},
+            )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error updating conversation name: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/feedback")
+class SubmitFeedback(Resource):
+    @api.expect(
+        api.model(
+            "FeedbackModel",
+            {
+                "question": fields.String(
+                    required=False, description="The user question"
+                ),
+                "answer": fields.String(required=False, description="The AI answer"),
+                "feedback": fields.String(required=True, description="User feedback"),
+                "question_index": fields.Integer(
+                    required=True,
+                    description="The question number in that particular conversation",
+                ),
+                "conversation_id": fields.String(
+                    required=True, description="id of the particular conversation"
+                ),
+                "api_key": fields.String(description="Optional API key"),
+            },
+        )
+    )
+    @api.doc(
+        description="Submit feedback for a conversation",
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        data = request.get_json()
+        required_fields = ["feedback", "conversation_id", "question_index"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            if data["feedback"] is None:
+                # Remove feedback and feedback_timestamp if feedback is null
+
+                conversations_collection.update_one(
+                    {
+                        "_id": ObjectId(data["conversation_id"]),
+                        "user": decoded_token.get("sub"),
+                        f"queries.{data['question_index']}": {"$exists": True},
+                    },
+                    {
+                        "$unset": {
+                            f"queries.{data['question_index']}.feedback": "",
+                            f"queries.{data['question_index']}.feedback_timestamp": "",
+                        }
+                    },
+                )
+            else:
+                # Set feedback and feedback_timestamp if feedback has a value
+
+                conversations_collection.update_one(
+                    {
+                        "_id": ObjectId(data["conversation_id"]),
+                        "user": decoded_token.get("sub"),
+                        f"queries.{data['question_index']}": {"$exists": True},
+                    },
+                    {
+                        "$set": {
+                            f"queries.{data['question_index']}.feedback": data[
+                                "feedback"
+                            ],
+                            f"queries.{data['question_index']}.feedback_timestamp": datetime.datetime.now(
+                                datetime.timezone.utc
+                            ),
+                        }
+                    },
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error submitting feedback: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)

application/api/user/models/__init__.py

@@ -0,0 +1,3 @@
+from .routes import models_ns
+
+__all__ = ["models_ns"]

application/api/user/models/routes.py

@@ -0,0 +1,25 @@
+from flask import current_app, jsonify, make_response
+from flask_restx import Namespace, Resource
+
+from application.core.model_settings import ModelRegistry
+
+models_ns = Namespace("models", description="Available models", path="/api")
+
+
+@models_ns.route("/models")
+class ModelsListResource(Resource):
+    def get(self):
+        """Get list of available models with their capabilities."""
+        try:
+            registry = ModelRegistry.get_instance()
+            models = registry.get_enabled_models()
+
+            response = {
+                "models": [model.to_dict() for model in models],
+                "default_model_id": registry.default_model_id,
+                "count": len(models),
+            }
+        except Exception as err:
+            current_app.logger.error(f"Error fetching models: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+        return make_response(jsonify(response), 200)

application/api/user/prompts/__init__.py

@@ -0,0 +1,5 @@
+"""Prompts module."""
+
+from .routes import prompts_ns
+
+__all__ = ["prompts_ns"]

application/api/user/prompts/routes.py

@@ -0,0 +1,191 @@
+"""Prompt management routes."""
+
+import os
+
+from bson.objectid import ObjectId
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from application.api import api
+from application.api.user.base import current_dir, prompts_collection
+from application.utils import check_required_fields
+
+prompts_ns = Namespace(
+    "prompts", description="Prompt management operations", path="/api"
+)
+
+
+@prompts_ns.route("/create_prompt")
+class CreatePrompt(Resource):
+    create_prompt_model = api.model(
+        "CreatePromptModel",
+        {
+            "content": fields.String(
+                required=True, description="Content of the prompt"
+            ),
+            "name": fields.String(required=True, description="Name of the prompt"),
+        },
+    )
+
+    @api.expect(create_prompt_model)
+    @api.doc(description="Create a new prompt")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        data = request.get_json()
+        required_fields = ["content", "name"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        user = decoded_token.get("sub")
+        try:
+
+            resp = prompts_collection.insert_one(
+                {
+                    "name": data["name"],
+                    "content": data["content"],
+                    "user": user,
+                }
+            )
+            new_id = str(resp.inserted_id)
+        except Exception as err:
+            current_app.logger.error(f"Error creating prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"id": new_id}), 200)
+
+
+@prompts_ns.route("/get_prompts")
+class GetPrompts(Resource):
+    @api.doc(description="Get all prompts for the user")
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            prompts = prompts_collection.find({"user": user})
+            list_prompts = [
+                {"id": "default", "name": "default", "type": "public"},
+                {"id": "creative", "name": "creative", "type": "public"},
+                {"id": "strict", "name": "strict", "type": "public"},
+            ]
+
+            for prompt in prompts:
+                list_prompts.append(
+                    {
+                        "id": str(prompt["_id"]),
+                        "name": prompt["name"],
+                        "type": "private",
+                    }
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving prompts: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify(list_prompts), 200)
+
+
+@prompts_ns.route("/get_single_prompt")
+class GetSinglePrompt(Resource):
+    @api.doc(params={"id": "ID of the prompt"}, description="Get a single prompt by ID")
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        prompt_id = request.args.get("id")
+        if not prompt_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        try:
+            if prompt_id == "default":
+                with open(
+                    os.path.join(current_dir, "prompts", "chat_combine_default.txt"),
+                    "r",
+                ) as f:
+                    chat_combine_template = f.read()
+                return make_response(jsonify({"content": chat_combine_template}), 200)
+            elif prompt_id == "creative":
+                with open(
+                    os.path.join(current_dir, "prompts", "chat_combine_creative.txt"),
+                    "r",
+                ) as f:
+                    chat_reduce_creative = f.read()
+                return make_response(jsonify({"content": chat_reduce_creative}), 200)
+            elif prompt_id == "strict":
+                with open(
+                    os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r"
+                ) as f:
+                    chat_reduce_strict = f.read()
+                return make_response(jsonify({"content": chat_reduce_strict}), 200)
+            prompt = prompts_collection.find_one(
+                {"_id": ObjectId(prompt_id), "user": user}
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"content": prompt["content"]}), 200)
+
+
+@prompts_ns.route("/delete_prompt")
+class DeletePrompt(Resource):
+    delete_prompt_model = api.model(
+        "DeletePromptModel",
+        {"id": fields.String(required=True, description="Prompt ID to delete")},
+    )
+
+    @api.expect(delete_prompt_model)
+    @api.doc(description="Delete a prompt by ID")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            prompts_collection.delete_one({"_id": ObjectId(data["id"]), "user": user})
+        except Exception as err:
+            current_app.logger.error(f"Error deleting prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@prompts_ns.route("/update_prompt")
+class UpdatePrompt(Resource):
+    update_prompt_model = api.model(
+        "UpdatePromptModel",
+        {
+            "id": fields.String(required=True, description="Prompt ID to update"),
+            "name": fields.String(required=True, description="New name of the prompt"),
+            "content": fields.String(
+                required=True, description="New content of the prompt"
+            ),
+        },
+    )
+
+    @api.expect(update_prompt_model)
+    @api.doc(description="Update an existing prompt")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "name", "content"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            prompts_collection.update_one(
+                {"_id": ObjectId(data["id"]), "user": user},
+                {"$set": {"name": data["name"], "content": data["content"]}},
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error updating prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)

application/api/user/routes.py

@@ -0,0 +1,52 @@
+"""
+Main user API routes - registers all namespace modules.
+"""
+
+from flask import Blueprint
+
+from application.api import api
+from .agents import agents_ns, agents_sharing_ns, agents_webhooks_ns
+
+from .analytics import analytics_ns
+from .attachments import attachments_ns
+from .conversations import conversations_ns
+from .models import models_ns
+from .prompts import prompts_ns
+from .sharing import sharing_ns
+from .sources import sources_chunks_ns, sources_ns, sources_upload_ns
+from .tools import tools_mcp_ns, tools_ns
+
+
+user = Blueprint("user", __name__)
+
+# Analytics
+api.add_namespace(analytics_ns)
+
+# Attachments
+api.add_namespace(attachments_ns)
+
+# Conversations
+api.add_namespace(conversations_ns)
+
+# Models
+api.add_namespace(models_ns)
+
+# Agents (main, sharing, webhooks)
+api.add_namespace(agents_ns)
+api.add_namespace(agents_sharing_ns)
+api.add_namespace(agents_webhooks_ns)
+
+# Prompts
+api.add_namespace(prompts_ns)
+
+# Sharing
+api.add_namespace(sharing_ns)
+
+# Sources (main, chunks, upload)
+api.add_namespace(sources_ns)
+api.add_namespace(sources_chunks_ns)
+api.add_namespace(sources_upload_ns)
+
+# Tools (main, MCP)
+api.add_namespace(tools_ns)
+api.add_namespace(tools_mcp_ns)

application/api/user/sharing/__init__.py

@@ -0,0 +1,5 @@
+"""Sharing module."""
+
+from .routes import sharing_ns
+
+__all__ = ["sharing_ns"]

application/api/user/sharing/routes.py

@@ -0,0 +1,289 @@
+"""Conversation sharing routes."""
+
+import uuid
+
+from bson.binary import Binary, UuidRepresentation
+from bson.dbref import DBRef
+from bson.objectid import ObjectId
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, inputs, Namespace, Resource
+
+from application.api import api
+from application.api.user.base import (
+    agents_collection,
+    attachments_collection,
+    conversations_collection,
+    shared_conversations_collections,
+)
+from application.utils import check_required_fields
+
+sharing_ns = Namespace(
+    "sharing", description="Conversation sharing operations", path="/api"
+)
+
+
+@sharing_ns.route("/share")
+class ShareConversation(Resource):
+    share_conversation_model = api.model(
+        "ShareConversationModel",
+        {
+            "conversation_id": fields.String(
+                required=True, description="Conversation ID"
+            ),
+            "user": fields.String(description="User ID (optional)"),
+            "prompt_id": fields.String(description="Prompt ID (optional)"),
+            "chunks": fields.Integer(description="Chunks count (optional)"),
+        },
+    )
+
+    @api.expect(share_conversation_model)
+    @api.doc(description="Share a conversation")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["conversation_id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        is_promptable = request.args.get("isPromptable", type=inputs.boolean)
+        if is_promptable is None:
+            return make_response(
+                jsonify({"success": False, "message": "isPromptable is required"}), 400
+            )
+        conversation_id = data["conversation_id"]
+
+        try:
+            conversation = conversations_collection.find_one(
+                {"_id": ObjectId(conversation_id)}
+            )
+            if conversation is None:
+                return make_response(
+                    jsonify(
+                        {
+                            "status": "error",
+                            "message": "Conversation does not exist",
+                        }
+                    ),
+                    404,
+                )
+            current_n_queries = len(conversation["queries"])
+            explicit_binary = Binary.from_uuid(
+                uuid.uuid4(), UuidRepresentation.STANDARD
+            )
+
+            if is_promptable:
+                prompt_id = data.get("prompt_id", "default")
+                chunks = data.get("chunks", "2")
+
+                name = conversation["name"] + "(shared)"
+                new_api_key_data = {
+                    "prompt_id": prompt_id,
+                    "chunks": chunks,
+                    "user": user,
+                }
+
+                if "source" in data and ObjectId.is_valid(data["source"]):
+                    new_api_key_data["source"] = DBRef(
+                        "sources", ObjectId(data["source"])
+                    )
+                if "retriever" in data:
+                    new_api_key_data["retriever"] = data["retriever"]
+                pre_existing_api_document = agents_collection.find_one(new_api_key_data)
+                if pre_existing_api_document:
+                    api_uuid = pre_existing_api_document["key"]
+                    pre_existing = shared_conversations_collections.find_one(
+                        {
+                            "conversation_id": ObjectId(conversation_id),
+                            "isPromptable": is_promptable,
+                            "first_n_queries": current_n_queries,
+                            "user": user,
+                            "api_key": api_uuid,
+                        }
+                    )
+                    if pre_existing is not None:
+                        return make_response(
+                            jsonify(
+                                {
+                                    "success": True,
+                                    "identifier": str(pre_existing["uuid"].as_uuid()),
+                                }
+                            ),
+                            200,
+                        )
+                    else:
+                        shared_conversations_collections.insert_one(
+                            {
+                                "uuid": explicit_binary,
+                                "conversation_id": ObjectId(conversation_id),
+                                "isPromptable": is_promptable,
+                                "first_n_queries": current_n_queries,
+                                "user": user,
+                                "api_key": api_uuid,
+                            }
+                        )
+                        return make_response(
+                            jsonify(
+                                {
+                                    "success": True,
+                                    "identifier": str(explicit_binary.as_uuid()),
+                                }
+                            ),
+                            201,
+                        )
+                else:
+                    api_uuid = str(uuid.uuid4())
+                    new_api_key_data["key"] = api_uuid
+                    new_api_key_data["name"] = name
+
+                    if "source" in data and ObjectId.is_valid(data["source"]):
+                        new_api_key_data["source"] = DBRef(
+                            "sources", ObjectId(data["source"])
+                        )
+                    if "retriever" in data:
+                        new_api_key_data["retriever"] = data["retriever"]
+                    agents_collection.insert_one(new_api_key_data)
+                    shared_conversations_collections.insert_one(
+                        {
+                            "uuid": explicit_binary,
+                            "conversation_id": ObjectId(conversation_id),
+                            "isPromptable": is_promptable,
+                            "first_n_queries": current_n_queries,
+                            "user": user,
+                            "api_key": api_uuid,
+                        }
+                    )
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": True,
+                                "identifier": str(explicit_binary.as_uuid()),
+                            }
+                        ),
+                        201,
+                    )
+            pre_existing = shared_conversations_collections.find_one(
+                {
+                    "conversation_id": ObjectId(conversation_id),
+                    "isPromptable": is_promptable,
+                    "first_n_queries": current_n_queries,
+                    "user": user,
+                }
+            )
+            if pre_existing is not None:
+                return make_response(
+                    jsonify(
+                        {
+                            "success": True,
+                            "identifier": str(pre_existing["uuid"].as_uuid()),
+                        }
+                    ),
+                    200,
+                )
+            else:
+                shared_conversations_collections.insert_one(
+                    {
+                        "uuid": explicit_binary,
+                        "conversation_id": ObjectId(conversation_id),
+                        "isPromptable": is_promptable,
+                        "first_n_queries": current_n_queries,
+                        "user": user,
+                    }
+                )
+                return make_response(
+                    jsonify(
+                        {"success": True, "identifier": str(explicit_binary.as_uuid())}
+                    ),
+                    201,
+                )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error sharing conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+
+
+@sharing_ns.route("/shared_conversation/<string:identifier>")
+class GetPubliclySharedConversations(Resource):
+    @api.doc(description="Get publicly shared conversations by identifier")
+    def get(self, identifier: str):
+        try:
+            query_uuid = Binary.from_uuid(
+                uuid.UUID(identifier), UuidRepresentation.STANDARD
+            )
+            shared = shared_conversations_collections.find_one({"uuid": query_uuid})
+            conversation_queries = []
+
+            if (
+                shared
+                and "conversation_id" in shared
+            ):
+                # conversation_id is now stored as an ObjectId, not a DBRef
+                conversation_id = shared["conversation_id"]
+                conversation = conversations_collection.find_one(
+                    {"_id": conversation_id}
+                )
+                if conversation is None:
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "error": "might have broken url or the conversation does not exist",
+                            }
+                        ),
+                        404,
+                    )
+                conversation_queries = conversation["queries"][
+                    : (shared["first_n_queries"])
+                ]
+
+                for query in conversation_queries:
+                    if "attachments" in query and query["attachments"]:
+                        attachment_details = []
+                        for attachment_id in query["attachments"]:
+                            try:
+                                attachment = attachments_collection.find_one(
+                                    {"_id": ObjectId(attachment_id)}
+                                )
+                                if attachment:
+                                    attachment_details.append(
+                                        {
+                                            "id": str(attachment["_id"]),
+                                            "fileName": attachment.get(
+                                                "filename", "Unknown file"
+                                            ),
+                                        }
+                                    )
+                            except Exception as e:
+                                current_app.logger.error(
+                                    f"Error retrieving attachment {attachment_id}: {e}",
+                                    exc_info=True,
+                                )
+                        query["attachments"] = attachment_details
+            else:
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "error": "might have broken url or the conversation does not exist",
+                        }
+                    ),
+                    404,
+                )
+            date = conversation["_id"].generation_time.isoformat()
+            res = {
+                "success": True,
+                "queries": conversation_queries,
+                "title": conversation["name"],
+                "timestamp": date,
+            }
+            if shared["isPromptable"] and "api_key" in shared:
+                res["api_key"] = shared["api_key"]
+            return make_response(jsonify(res), 200)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error getting shared conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)

application/api/user/sources/__init__.py

@@ -0,0 +1,7 @@
+"""Sources module."""
+
+from .chunks import sources_chunks_ns
+from .routes import sources_ns
+from .upload import sources_upload_ns
+
+__all__ = ["sources_ns", "sources_chunks_ns", "sources_upload_ns"]