fix: comments in settings

feat: faster ebook parsing

.env-template

@@ -3,6 +3,14 @@ LLM_NAME=docsgpt
 VITE_API_STREAMING=true
 INTERNAL_KEY=<internal key for worker-to-backend authentication>
 
+# Provider-specific API keys (optional - use these to enable multiple providers)
+# OPENAI_API_KEY=<your-openai-api-key>
+# ANTHROPIC_API_KEY=<your-anthropic-api-key>
+# GOOGLE_API_KEY=<your-google-api-key>
+# GROQ_API_KEY=<your-groq-api-key>
+# NOVITA_API_KEY=<your-novita-api-key>
+# OPEN_ROUTER_API_KEY=<your-openrouter-api-key>
+
 # Remote Embeddings (Optional - for using a remote embeddings API instead of local SentenceTransformer)
 # When set, the app will use the remote API and won't load SentenceTransformer (saves RAM)
 EMBEDDINGS_BASE_URL=
@@ -26,3 +34,9 @@ MICROSOFT_TENANT_ID=your-azure-ad-tenant-id
 #or "https://login.microsoftonline.com/contoso.onmicrosoft.com".
 #Alternatively, use "https://login.microsoftonline.com/common" for multi-tenant app.
 MICROSOFT_AUTHORITY=https://{tenantId}.ciamlogin.com/{tenantId}
+
+# User-data Postgres DB (Phase 0 of the MongoDB→Postgres migration).
+# Standard Postgres URI — `postgres://` and `postgresql://` both work.
+# Leave unset while the migration is still being rolled out; the app will
+# fall back to MongoDB for user data until POSTGRES_URI is configured.
+# POSTGRES_URI=postgresql://docsgpt:docsgpt@localhost:5432/docsgpt

.github/styles/config/vocabularies/DocsGPT/accept.txt

@@ -44,3 +44,8 @@ boolean
 bool
 hardcode
 EOL
+Postgres
+Supabase
+config
+backfill
+backfills

.gitignore

@@ -108,6 +108,8 @@ celerybeat.pid
 # Environments
 .env
 .venv
+# Machine-specific Claude Code guidance (see CLAUDE.md preamble)
+CLAUDE.md
 env/
 venv/
 ENV/

.vale.ini

@@ -1,5 +1,7 @@
 MinAlertLevel = warning
 StylesPath = .github/styles
+Vocab = DocsGPT
 
 [*.{md,mdx}]
 BasedOnStyles = DocsGPT
+

README.md

@@ -29,7 +29,7 @@
 
 <div align="center">
   <br>
-<img src="https://d3dg1063dc54p9.cloudfront.net/videos/demov7.gif" alt="video-example-of-docs-gpt" width="800" height="450">
+<img src="https://d3dg1063dc54p9.cloudfront.net/videos/demo-26.gif" alt="video-example-of-docs-gpt" width="800" height="480">
 </div>
 <h3 align="left">
   <strong>Key Features:</strong>

SECURITY.md

@@ -2,13 +2,21 @@
 
 ## Supported Versions
 
-Supported Versions:
-
-Currently, we support security patches by committing changes and bumping the version published on Github.
+Security patches target the latest release and the `main` branch. We recommend always running the most recent version.
 
 ## Reporting a Vulnerability
 
-Found a vulnerability? Please email us:
+Preferred method: use GitHub's private vulnerability reporting flow:
+https://github.com/arc53/DocsGPT/security
 
-security@arc53.com
+Then click **Report a vulnerability**.
+
+
+Alternatively, email us at: security@arc53.com
+
+We aim to acknowledge reports within 48 hours.
+
+## Incident Handling
+
+Arc53 maintains internal incident response procedures. If you believe an active exploit is occurring, include **URGENT** in your report subject line.
 

application/agents/base.py

@@ -1,7 +1,8 @@
+import json
 import logging
 import uuid
 from abc import ABC, abstractmethod
-from typing import Dict, Generator, List, Optional
+from typing import Any, Dict, Generator, List, Optional
 
 from application.agents.tool_executor import ToolExecutor
 from application.core.json_schema_utils import (
@@ -9,6 +10,7 @@ from application.core.json_schema_utils import (
     normalize_json_schema_payload,
 )
 from application.core.settings import settings
+from application.llm.handlers.base import ToolCall
 from application.llm.handlers.handler_creator import LLMHandlerCreator
 from application.llm.llm_creator import LLMCreator
 from application.logging import build_stack_data, log_activity, LogContext
@@ -113,6 +115,153 @@ class BaseAgent(ABC):
     ) -> Generator[Dict, None, None]:
         pass
 
+    def gen_continuation(
+        self,
+        messages: List[Dict],
+        tools_dict: Dict,
+        pending_tool_calls: List[Dict],
+        tool_actions: List[Dict],
+    ) -> Generator[Dict, None, None]:
+        """Resume generation after tool actions are resolved.
+
+        Processes the client-provided *tool_actions* (approvals, denials,
+        or client-side results), appends the resulting messages, then
+        hands back to the LLM to continue the conversation.
+
+        Args:
+            messages: The saved messages array from the pause point.
+            tools_dict: The saved tools dictionary.
+            pending_tool_calls: The pending tool call descriptors from the pause.
+            tool_actions: Client-provided actions resolving the pending calls.
+        """
+        self._prepare_tools(tools_dict)
+
+        actions_by_id = {a["call_id"]: a for a in tool_actions}
+
+        # Build a single assistant message containing all tool calls so
+        # the message history matches the format LLM providers expect
+        # (one assistant message with N tool_calls, followed by N tool results).
+        tc_objects: List[Dict[str, Any]] = []
+        for pending in pending_tool_calls:
+            call_id = pending["call_id"]
+            args = pending["arguments"]
+            args_str = (
+                json.dumps(args) if isinstance(args, dict) else (args or "{}")
+            )
+            tc_obj: Dict[str, Any] = {
+                "id": call_id,
+                "type": "function",
+                "function": {
+                    "name": pending["name"],
+                    "arguments": args_str,
+                },
+            }
+            if pending.get("thought_signature"):
+                tc_obj["thought_signature"] = pending["thought_signature"]
+            tc_objects.append(tc_obj)
+
+        messages.append({
+            "role": "assistant",
+            "content": None,
+            "tool_calls": tc_objects,
+        })
+
+        # Now process each pending call and append tool result messages
+        for pending in pending_tool_calls:
+            call_id = pending["call_id"]
+            args = pending["arguments"]
+            action = actions_by_id.get(call_id)
+            if not action:
+                action = {
+                    "call_id": call_id,
+                    "decision": "denied",
+                    "comment": "No response provided",
+                }
+
+            if action.get("decision") == "approved":
+                # Execute the tool server-side
+                tc = ToolCall(
+                    id=call_id,
+                    name=pending["name"],
+                    arguments=(
+                        json.dumps(args) if isinstance(args, dict) else args
+                    ),
+                )
+                tool_gen = self._execute_tool_action(tools_dict, tc)
+                tool_response = None
+                while True:
+                    try:
+                        event = next(tool_gen)
+                        yield event
+                    except StopIteration as e:
+                        tool_response, _ = e.value
+                        break
+                messages.append(
+                    self.llm_handler.create_tool_message(tc, tool_response)
+                )
+
+            elif action.get("decision") == "denied":
+                comment = action.get("comment", "")
+                denial = (
+                    f"Tool execution denied by user. Reason: {comment}"
+                    if comment
+                    else "Tool execution denied by user."
+                )
+                tc = ToolCall(
+                    id=call_id, name=pending["name"], arguments=args
+                )
+                messages.append(
+                    self.llm_handler.create_tool_message(tc, denial)
+                )
+                yield {
+                    "type": "tool_call",
+                    "data": {
+                        "tool_name": pending.get("tool_name", "unknown"),
+                        "call_id": call_id,
+                        "action_name": pending.get("llm_name", pending["name"]),
+                        "arguments": args,
+                        "status": "denied",
+                    },
+                }
+
+            elif "result" in action:
+                result = action["result"]
+                result_str = (
+                    json.dumps(result)
+                    if not isinstance(result, str)
+                    else result
+                )
+                tc = ToolCall(
+                    id=call_id, name=pending["name"], arguments=args
+                )
+                messages.append(
+                    self.llm_handler.create_tool_message(tc, result_str)
+                )
+                yield {
+                    "type": "tool_call",
+                    "data": {
+                        "tool_name": pending.get("tool_name", "unknown"),
+                        "call_id": call_id,
+                        "action_name": pending.get("llm_name", pending["name"]),
+                        "arguments": args,
+                        "result": (
+                            result_str[:50] + "..."
+                            if len(result_str) > 50
+                            else result_str
+                        ),
+                        "status": "completed",
+                    },
+                }
+
+        # Resume the LLM loop with the updated messages
+        llm_response = self._llm_gen(messages)
+        yield from self._handle_response(
+            llm_response, tools_dict, messages, None
+        )
+
+        yield {"sources": self.retrieved_docs}
+        yield {"tool_calls": self._get_truncated_tool_calls()}
+
     # ---- Tool delegation (thin wrappers around ToolExecutor) ----
 
     @property
@@ -267,28 +416,35 @@ class BaseAgent(ABC):
             if "tool_calls" in i:
                 for tool_call in i["tool_calls"]:
                     call_id = tool_call.get("call_id") or str(uuid.uuid4())
-
-                    function_call_dict = {
-                        "function_call": {
-                            "name": tool_call.get("action_name"),
-                            "args": tool_call.get("arguments"),
-                            "call_id": call_id,
-                        }
-                    }
-                    function_response_dict = {
-                        "function_response": {
-                            "name": tool_call.get("action_name"),
-                            "response": {"result": tool_call.get("result")},
-                            "call_id": call_id,
-                        }
-                    }
-
-                    messages.append(
-                        {"role": "assistant", "content": [function_call_dict]}
+                    args = tool_call.get("arguments")
+                    args_str = (
+                        json.dumps(args)
+                        if isinstance(args, dict)
+                        else (args or "{}")
                     )
-                    messages.append(
-                        {"role": "tool", "content": [function_response_dict]}
+                    messages.append({
+                        "role": "assistant",
+                        "content": None,
+                        "tool_calls": [{
+                            "id": call_id,
+                            "type": "function",
+                            "function": {
+                                "name": tool_call.get("action_name", ""),
+                                "arguments": args_str,
+                            },
+                        }],
+                    })
+                    result = tool_call.get("result")
+                    result_str = (
+                        json.dumps(result)
+                        if not isinstance(result, str)
+                        else (result or "")
                     )
+                    messages.append({
+                        "role": "tool",
+                        "tool_call_id": call_id,
+                        "content": result_str,
+                    })
         messages.append({"role": "user", "content": query})
         return messages
 

application/agents/research_agent.py

@@ -593,16 +593,22 @@ class ResearchAgent(BaseAgent):
                     )
                     result = result_str
 
-            function_call_content = {
-                "function_call": {
-                    "name": call.name,
-                    "args": call.arguments,
-                    "call_id": call_id,
-                }
-            }
-            messages.append(
-                {"role": "assistant", "content": [function_call_content]}
+            import json as _json
+
+            args_str = (
+                _json.dumps(call.arguments)
+                if isinstance(call.arguments, dict)
+                else call.arguments
             )
+            messages.append({
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [{
+                    "id": call_id,
+                    "type": "function",
+                    "function": {"name": call.name, "arguments": args_str},
+                }],
+            })
             tool_message = self.llm_handler.create_tool_message(call, result)
             messages.append(tool_message)
 

application/agents/tool_executor.py

@@ -1,6 +1,7 @@
 import logging
 import uuid
-from typing import Dict, List, Optional
+from collections import Counter
+from typing import Dict, List, Optional, Tuple
 
 from bson.objectid import ObjectId
 
@@ -31,12 +32,23 @@ class ToolExecutor:
         self.tool_calls: List[Dict] = []
         self._loaded_tools: Dict[str, object] = {}
         self.conversation_id: Optional[str] = None
+        self.client_tools: Optional[List[Dict]] = None
+        self._name_to_tool: Dict[str, Tuple[str, str]] = {}
+        self._tool_to_name: Dict[Tuple[str, str], str] = {}
 
     def get_tools(self) -> Dict[str, Dict]:
-        """Load tool configs from DB based on user context."""
+        """Load tool configs from DB based on user context.
+
+        If *client_tools* have been set on this executor, they are
+        automatically merged into the returned dict.
+        """
         if self.user_api_key:
-            return self._get_tools_by_api_key(self.user_api_key)
-        return self._get_user_tools(self.user or "local")
+            tools = self._get_tools_by_api_key(self.user_api_key)
+        else:
+            tools = self._get_user_tools(self.user or "local")
+        if self.client_tools:
+            self.merge_client_tools(tools, self.client_tools)
+        return tools
 
     def _get_tools_by_api_key(self, api_key: str) -> Dict[str, Dict]:
         mongo = MongoDB.get_client()
@@ -65,29 +77,123 @@ class ToolExecutor:
         user_tools = list(user_tools)
         return {str(i): tool for i, tool in enumerate(user_tools)}
 
-    def prepare_tools_for_llm(self, tools_dict: Dict) -> List[Dict]:
-        """Convert tool configs to LLM function schemas."""
-        return [
-            {
-                "type": "function",
-                "function": {
-                    "name": f"{action['name']}_{tool_id}",
-                    "description": action["description"],
-                    "parameters": self._build_tool_parameters(action),
-                },
+    def merge_client_tools(
+        self, tools_dict: Dict, client_tools: List[Dict]
+    ) -> Dict:
+        """Merge client-provided tool definitions into tools_dict.
+
+        Client tools use the standard function-calling format::
+
+            [{"type": "function", "function": {"name": "get_weather",
+              "description": "...", "parameters": {...}}}]
+
+        They are stored in *tools_dict* with ``client_side: True`` so that
+        :meth:`check_pause` returns a pause signal instead of trying to
+        execute them server-side.
+
+        Args:
+            tools_dict: The mutable server tools dict (will be modified in place).
+            client_tools: List of tool definitions in function-calling format.
+
+        Returns:
+            The updated *tools_dict* (same reference, for convenience).
+        """
+        for i, ct in enumerate(client_tools):
+            func = ct.get("function", ct)  # tolerate bare {"name":..} too
+            name = func.get("name", f"clienttool{i}")
+            tool_id = f"ct{i}"
+
+            tools_dict[tool_id] = {
+                "name": name,
+                "client_side": True,
+                "actions": [
+                    {
+                        "name": name,
+                        "description": func.get("description", ""),
+                        "active": True,
+                        "parameters": func.get("parameters", {}),
+                    }
+                ],
             }
-            for tool_id, tool in tools_dict.items()
-            if (
-                (tool["name"] == "api_tool" and "actions" in tool.get("config", {}))
-                or (tool["name"] != "api_tool" and "actions" in tool)
-            )
-            for action in (
+        return tools_dict
+
+    def prepare_tools_for_llm(self, tools_dict: Dict) -> List[Dict]:
+        """Convert tool configs to LLM function schemas.
+
+        Action names are kept clean for the LLM:
+        - Unique action names appear as-is (e.g. ``get_weather``).
+        - Duplicate action names get numbered suffixes (e.g. ``search_1``,
+          ``search_2``).
+
+        A reverse mapping is stored in ``_name_to_tool`` so that tool calls
+        can be routed back to the correct ``(tool_id, action_name)`` without
+        brittle string splitting.
+        """
+        # Pass 1: collect entries and count action name occurrences
+        entries: List[Tuple[str, str, Dict, bool]] = []  # (tool_id, action_name, action, is_client)
+        name_counts: Counter = Counter()
+
+        for tool_id, tool in tools_dict.items():
+            is_api = tool["name"] == "api_tool"
+            is_client = tool.get("client_side", False)
+
+            if is_api and "actions" not in tool.get("config", {}):
+                continue
+            if not is_api and "actions" not in tool:
+                continue
+
+            actions = (
                 tool["config"]["actions"].values()
-                if tool["name"] == "api_tool"
+                if is_api
                 else tool["actions"]
             )
-            if action.get("active", True)
-        ]
+
+            for action in actions:
+                if not action.get("active", True):
+                    continue
+                entries.append((tool_id, action["name"], action, is_client))
+                name_counts[action["name"]] += 1
+
+        # Pass 2: assign LLM-visible names and build mappings
+        self._name_to_tool = {}
+        self._tool_to_name = {}
+        collision_counters: Dict[str, int] = {}
+        all_llm_names: set = set()
+
+        result = []
+        for tool_id, action_name, action, is_client in entries:
+            if name_counts[action_name] == 1:
+                llm_name = action_name
+            else:
+                counter = collision_counters.get(action_name, 1)
+                candidate = f"{action_name}_{counter}"
+                # Skip if candidate collides with a unique action name
+                while candidate in all_llm_names or (
+                    candidate in name_counts and name_counts[candidate] == 1
+                ):
+                    counter += 1
+                    candidate = f"{action_name}_{counter}"
+                collision_counters[action_name] = counter + 1
+                llm_name = candidate
+
+            all_llm_names.add(llm_name)
+            self._name_to_tool[llm_name] = (tool_id, action_name)
+            self._tool_to_name[(tool_id, action_name)] = llm_name
+
+            if is_client:
+                params = action.get("parameters", {})
+            else:
+                params = self._build_tool_parameters(action)
+
+            result.append({
+                "type": "function",
+                "function": {
+                    "name": llm_name,
+                    "description": action.get("description", ""),
+                    "parameters": params,
+                },
+            })
+        return result
 
     def _build_tool_parameters(self, action: Dict) -> Dict:
         params = {"type": "object", "properties": {}, "required": []}
@@ -104,23 +210,81 @@ class ToolExecutor:
                             params["required"].append(k)
         return params
 
+    def check_pause(
+        self, tools_dict: Dict, call, llm_class_name: str
+    ) -> Optional[Dict]:
+        """Check if a tool call requires pausing for approval or client execution.
+
+        Returns a dict describing the pending action if pause is needed, None otherwise.
+        """
+        parser = ToolActionParser(llm_class_name, name_mapping=self._name_to_tool)
+        tool_id, action_name, call_args = parser.parse_args(call)
+        call_id = getattr(call, "id", None) or str(uuid.uuid4())
+        llm_name = getattr(call, "name", "")
+
+        if tool_id is None or action_name is None or tool_id not in tools_dict:
+            return None  # Will be handled as error by execute()
+
+        tool_data = tools_dict[tool_id]
+
+        # Client-side tools
+        if tool_data.get("client_side"):
+            return {
+                "call_id": call_id,
+                "name": llm_name,
+                "tool_name": tool_data.get("name", "unknown"),
+                "tool_id": tool_id,
+                "action_name": action_name,
+                "llm_name": llm_name,
+                "arguments": call_args if isinstance(call_args, dict) else {},
+                "pause_type": "requires_client_execution",
+                "thought_signature": getattr(call, "thought_signature", None),
+            }
+
+        # Approval required
+        if tool_data["name"] == "api_tool":
+            action_data = tool_data.get("config", {}).get("actions", {}).get(
+                action_name, {}
+            )
+        else:
+            action_data = next(
+                (a for a in tool_data.get("actions", []) if a["name"] == action_name),
+                {},
+            )
+
+        if action_data.get("require_approval"):
+            return {
+                "call_id": call_id,
+                "name": llm_name,
+                "tool_name": tool_data.get("name", "unknown"),
+                "tool_id": tool_id,
+                "action_name": action_name,
+                "llm_name": llm_name,
+                "arguments": call_args if isinstance(call_args, dict) else {},
+                "pause_type": "awaiting_approval",
+                "thought_signature": getattr(call, "thought_signature", None),
+            }
+
+        return None
+
     def execute(self, tools_dict: Dict, call, llm_class_name: str):
         """Execute a tool call. Yields status events, returns (result, call_id)."""
-        parser = ToolActionParser(llm_class_name)
+        parser = ToolActionParser(llm_class_name, name_mapping=self._name_to_tool)
         tool_id, action_name, call_args = parser.parse_args(call)
+        llm_name = getattr(call, "name", "unknown")
 
         call_id = getattr(call, "id", None) or str(uuid.uuid4())
 
         if tool_id is None or action_name is None:
-            error_message = f"Error: Failed to parse LLM tool call. Tool name: {getattr(call, 'name', 'unknown')}"
+            error_message = f"Error: Failed to parse LLM tool call. Tool name: {llm_name}"
             logger.error(error_message)
 
             tool_call_data = {
                 "tool_name": "unknown",
                 "call_id": call_id,
-                "action_name": getattr(call, "name", "unknown"),
+                "action_name": llm_name,
                 "arguments": call_args or {},
-                "result": f"Failed to parse tool call. Invalid tool name format: {getattr(call, 'name', 'unknown')}",
+                "result": f"Failed to parse tool call. Invalid tool name format: {llm_name}",
             }
             yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
             self.tool_calls.append(tool_call_data)
@@ -133,7 +297,7 @@ class ToolExecutor:
             tool_call_data = {
                 "tool_name": "unknown",
                 "call_id": call_id,
-                "action_name": f"{action_name}_{tool_id}",
+                "action_name": llm_name,
                 "arguments": call_args,
                 "result": f"Tool with ID {tool_id} not found. Available tools: {list(tools_dict.keys())}",
             }
@@ -144,7 +308,7 @@ class ToolExecutor:
         tool_call_data = {
             "tool_name": tools_dict[tool_id]["name"],
             "call_id": call_id,
-            "action_name": f"{action_name}_{tool_id}",
+            "action_name": llm_name,
             "arguments": call_args,
         }
         yield {"type": "tool_call", "data": {**tool_call_data, "status": "pending"}}
@@ -185,7 +349,10 @@ class ToolExecutor:
                     target_dict[param] = value
 
         # Load tool (with caching)
-        tool = self._get_or_load_tool(tool_data, tool_id, action_name)
+        tool = self._get_or_load_tool(
+            tool_data, tool_id, action_name,
+            headers=headers, query_params=query_params,
+        )
 
         resolved_arguments = (
             {"query_params": query_params, "headers": headers, "body": body}
@@ -238,7 +405,10 @@ class ToolExecutor:
 
         return result, call_id
 
-    def _get_or_load_tool(self, tool_data: Dict, tool_id: str, action_name: str):
+    def _get_or_load_tool(
+        self, tool_data: Dict, tool_id: str, action_name: str,
+        headers: Optional[Dict] = None, query_params: Optional[Dict] = None,
+    ):
         """Load a tool, using cache when possible."""
         cache_key = f"{tool_data['name']}:{tool_id}:{self.user or ''}"
         if cache_key in self._loaded_tools:
@@ -251,8 +421,8 @@ class ToolExecutor:
             tool_config = {
                 "url": action_config["url"],
                 "method": action_config["method"],
-                "headers": {},
-                "query_params": {},
+                "headers": headers or {},
+                "query_params": query_params or {},
             }
             if "body_content_type" in action_config:
                 tool_config["body_content_type"] = action_config.get(

application/agents/tools/base.py

@@ -2,6 +2,8 @@ from abc import ABC, abstractmethod
 
 
 class Tool(ABC):
+    internal: bool = False
+
     @abstractmethod
     def execute_action(self, action_name: str, **kwargs):
         pass

application/agents/tools/internal_search.py

@@ -20,6 +20,8 @@ class InternalSearchTool(Tool):
     - list_files action: browse the file/folder structure
     """
 
+    internal = True
+
     def __init__(self, config: Dict):
         self.config = config
         self.retrieved_docs: List[Dict] = []

application/agents/tools/mcp_tool.py

@@ -24,6 +24,7 @@ from application.api.user.tasks import mcp_oauth_status_task, mcp_oauth_task
 from application.cache import get_redis_instance
 from application.core.mongo_db import MongoDB
 from application.core.settings import settings
+from application.core.url_validation import SSRFError, validate_url
 from application.security.encryption import decrypt_credentials
 
 logger = logging.getLogger(__name__)
@@ -61,7 +62,8 @@ class MCPTool(Tool):
         """
         self.config = config
         self.user_id = user_id
-        self.server_url = config.get("server_url", "")
+        raw_url = config.get("server_url", "")
+        self.server_url = self._validate_server_url(raw_url) if raw_url else ""
         self.transport_type = config.get("transport_type", "auto")
         self.auth_type = config.get("auth_type", "none")
         self.timeout = config.get("timeout", 30)
@@ -87,6 +89,18 @@ class MCPTool(Tool):
         if self.server_url and self.auth_type != "oauth":
             self._setup_client()
 
+    @staticmethod
+    def _validate_server_url(server_url: str) -> str:
+        """Validate server_url to prevent SSRF to internal networks.
+
+        Raises:
+            ValueError: If the URL points to a private/internal address.
+        """
+        try:
+            return validate_url(server_url)
+        except SSRFError as exc:
+            raise ValueError(f"Invalid MCP server URL: {exc}") from exc
+
     def _resolve_redirect_uri(self, configured_redirect_uri: Optional[str]) -> str:
         if configured_redirect_uri:
             return configured_redirect_uri.rstrip("/")
@@ -108,8 +122,9 @@ class MCPTool(Tool):
         auth_key = ""
         if self.auth_type == "oauth":
             scopes_str = ",".join(self.oauth_scopes) if self.oauth_scopes else "none"
+            oauth_identity = self.user_id or self.oauth_task_id or "anonymous"
             auth_key = (
-                f"oauth:{self.oauth_client_name}:{scopes_str}:{self.redirect_uri}"
+                f"oauth:{oauth_identity}:{self.oauth_client_name}:{scopes_str}:{self.redirect_uri}"
             )
         elif self.auth_type in ["bearer"]:
             token = self.auth_credentials.get(

application/agents/tools/postgres.py

@@ -1,6 +1,6 @@
 import logging
 
-import psycopg2
+import psycopg
 
 from application.agents.tools.base import Tool
 
@@ -33,7 +33,7 @@ class PostgresTool(Tool):
         """
         conn = None
         try:
-            conn = psycopg2.connect(self.connection_string)
+            conn = psycopg.connect(self.connection_string)
             cur = conn.cursor()
             cur.execute(sql_query)
             conn.commit()
@@ -60,7 +60,7 @@ class PostgresTool(Tool):
                 "response_data": response_data,
             }
 
-        except psycopg2.Error as e:
+        except psycopg.Error as e:
             error_message = f"Database error: {e}"
             logger.error("PostgreSQL execute_sql error: %s", e)
             return {
@@ -78,7 +78,7 @@ class PostgresTool(Tool):
         """
         conn = None
         try:
-            conn = psycopg2.connect(self.connection_string)
+            conn = psycopg.connect(self.connection_string)
             cur = conn.cursor()
 
             cur.execute(
@@ -120,7 +120,7 @@ class PostgresTool(Tool):
                 "schema": schema_data,
             }
 
-        except psycopg2.Error as e:
+        except psycopg.Error as e:
             error_message = f"Database error: {e}"
             logger.error("PostgreSQL get_schema error: %s", e)
             return {

application/agents/tools/think.py

@@ -36,6 +36,8 @@ class ThinkTool(Tool):
     The reasoning content is captured in tool_call data for transparency.
     """
 
+    internal = True
+
     def __init__(self, config=None):
         pass
 

application/agents/tools/tool_action_parser.py

@@ -5,8 +5,9 @@ logger = logging.getLogger(__name__)
 
 
 class ToolActionParser:
-    def __init__(self, llm_type):
+    def __init__(self, llm_type, name_mapping=None):
         self.llm_type = llm_type
+        self.name_mapping = name_mapping
         self.parsers = {
             "OpenAILLM": self._parse_openai_llm,
             "GoogleLLM": self._parse_google_llm,
@@ -16,22 +17,33 @@ class ToolActionParser:
         parser = self.parsers.get(self.llm_type, self._parse_openai_llm)
         return parser(call)
 
+    def _resolve_via_mapping(self, call_name):
+        """Look up (tool_id, action_name) from the name mapping if available."""
+        if self.name_mapping and call_name in self.name_mapping:
+            return self.name_mapping[call_name]
+        return None
+
     def _parse_openai_llm(self, call):
         try:
             call_args = json.loads(call.arguments)
+
+            resolved = self._resolve_via_mapping(call.name)
+            if resolved:
+                return resolved[0], resolved[1], call_args
+
+            # Fallback: legacy split on "_" for backward compatibility
             tool_parts = call.name.split("_")
 
-            # If the tool name doesn't contain an underscore, it's likely a hallucinated tool
             if len(tool_parts) < 2:
                 logger.warning(
-                    f"Invalid tool name format: {call.name}. Expected format: action_name_tool_id"
+                    f"Invalid tool name format: {call.name}. "
+                    "Could not resolve via mapping or legacy parsing."
                 )
                 return None, None, None
 
             tool_id = tool_parts[-1]
             action_name = "_".join(tool_parts[:-1])
 
-            # Validate that tool_id looks like a numerical ID
             if not tool_id.isdigit():
                 logger.warning(
                     f"Tool ID '{tool_id}' is not numerical. This might be a hallucinated tool call."
@@ -45,19 +57,24 @@ class ToolActionParser:
     def _parse_google_llm(self, call):
         try:
             call_args = call.arguments
+
+            resolved = self._resolve_via_mapping(call.name)
+            if resolved:
+                return resolved[0], resolved[1], call_args
+
+            # Fallback: legacy split on "_" for backward compatibility
             tool_parts = call.name.split("_")
 
-            # If the tool name doesn't contain an underscore, it's likely a hallucinated tool
             if len(tool_parts) < 2:
                 logger.warning(
-                    f"Invalid tool name format: {call.name}. Expected format: action_name_tool_id"
+                    f"Invalid tool name format: {call.name}. "
+                    "Could not resolve via mapping or legacy parsing."
                 )
                 return None, None, None
 
             tool_id = tool_parts[-1]
             action_name = "_".join(tool_parts[:-1])
 
-            # Validate that tool_id looks like a numerical ID
             if not tool_id.isdigit():
                 logger.warning(
                     f"Tool ID '{tool_id}' is not numerical. This might be a hallucinated tool call."

application/agents/tools/tool_manager.py

@@ -19,7 +19,7 @@ class ToolManager:
                 continue
             module = importlib.import_module(f"application.agents.tools.{name}")
             for member_name, obj in inspect.getmembers(module, inspect.isclass):
-                if issubclass(obj, Tool) and obj is not Tool:
+                if issubclass(obj, Tool) and obj is not Tool and not obj.internal:
                     tool_config = self.config.get(name, {})
                     self.tools[name] = obj(tool_config)
 

application/alembic.ini

@@ -0,0 +1,52 @@
+# Alembic configuration for the DocsGPT user-data Postgres database.
+#
+# The SQLAlchemy URL is deliberately NOT set here — env.py reads it from
+# ``application.core.settings.settings.POSTGRES_URI`` so the same config
+# source serves the running app and migrations. To run from the project
+# root::
+#
+#     alembic -c application/alembic.ini upgrade head
+
+[alembic]
+script_location = %(here)s/alembic
+prepend_sys_path = ..
+version_path_separator = os
+
+# sqlalchemy.url is intentionally left blank — env.py supplies it.
+sqlalchemy.url =
+
+[post_write_hooks]
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

application/alembic/env.py

@@ -0,0 +1,82 @@
+"""Alembic environment for the DocsGPT user-data Postgres database.
+
+The URL is pulled from ``application.core.settings`` rather than
+``alembic.ini`` so that a single ``POSTGRES_URI`` env var drives both the
+running app and ``alembic`` CLI invocations.
+"""
+
+import sys
+from logging.config import fileConfig
+from pathlib import Path
+
+# Make the project root importable regardless of cwd. env.py lives at
+# <repo>/application/alembic/env.py, so parents[2] is the repo root.
+_PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(_PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(_PROJECT_ROOT))
+
+from alembic import context  # noqa: E402
+from sqlalchemy import engine_from_config, pool  # noqa: E402
+
+from application.core.settings import settings  # noqa: E402
+from application.storage.db.models import metadata as target_metadata  # noqa: E402
+
+config = context.config
+
+# Populate the runtime URL from settings.
+if settings.POSTGRES_URI:
+    config.set_main_option("sqlalchemy.url", settings.POSTGRES_URI)
+
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode (emits SQL without a live DB)."""
+    url = config.get_main_option("sqlalchemy.url")
+    if not url:
+        raise RuntimeError(
+            "POSTGRES_URI is not configured. Set it in your .env to a "
+            "psycopg3 URI such as "
+            "'postgresql+psycopg://user:pass@host:5432/docsgpt'."
+        )
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        compare_type=True,
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode against a live connection."""
+    if not config.get_main_option("sqlalchemy.url"):
+        raise RuntimeError(
+            "POSTGRES_URI is not configured. Set it in your .env to a "
+            "psycopg3 URI such as "
+            "'postgresql+psycopg://user:pass@host:5432/docsgpt'."
+        )
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+        future=True,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+        )
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

application/alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

application/alembic/versions/0001_initial.py

@@ -0,0 +1,462 @@
+"""0001 initial schema — user-level tables migrated from MongoDB.
+
+Creates every table described in §2.2 of ``migration-postgres.md``: tiers 1,
+2, and 3 in one shot. The schema is small enough that splitting the baseline
+across multiple revisions would only cost clarity.
+
+Subsequent migrations will add columns / tables incrementally. This file is
+hand-written raw DDL rather than Core ``op.create_table`` calls because the
+DDL uses several Postgres-specific features (``CITEXT``, partial indexes,
+``text_pattern_ops``, JSONB defaults) that are clearer in SQL than in
+Alembic's Python API.
+
+Revision ID: 0001_initial
+Revises:
+Create Date: 2026-04-10
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "0001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ------------------------------------------------------------------
+    # Extensions
+    # ------------------------------------------------------------------
+    op.execute('CREATE EXTENSION IF NOT EXISTS "pgcrypto";')
+    op.execute('CREATE EXTENSION IF NOT EXISTS "citext";')
+
+    # ------------------------------------------------------------------
+    # Tier 1: leaf tables, no FKs into other migrated tables
+    # ------------------------------------------------------------------
+    op.execute("""
+        CREATE TABLE users (
+            id                UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id           TEXT NOT NULL UNIQUE,
+            agent_preferences JSONB NOT NULL
+                              DEFAULT '{"pinned": [], "shared_with_me": []}'::jsonb,
+            created_at        TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at        TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX users_user_id_idx ON users (user_id);")
+
+    op.execute("""
+        CREATE TABLE prompts (
+            id          UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id     TEXT NOT NULL,
+            name        TEXT NOT NULL,
+            content     TEXT NOT NULL,
+            created_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at  TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX prompts_user_id_idx ON prompts (user_id);")
+
+    op.execute("""
+        CREATE TABLE user_tools (
+            id           UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id      TEXT NOT NULL,
+            name         TEXT NOT NULL,
+            custom_name  TEXT,
+            display_name TEXT,
+            config       JSONB NOT NULL DEFAULT '{}'::jsonb,
+            created_at   TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at   TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX user_tools_user_id_idx ON user_tools (user_id);")
+
+    op.execute("""
+        CREATE TABLE token_usage (
+            id               BIGSERIAL PRIMARY KEY,
+            user_id          TEXT,
+            api_key          TEXT,
+            agent_id         UUID,                      -- FK added later in this migration
+            prompt_tokens    INTEGER NOT NULL DEFAULT 0,
+            generated_tokens INTEGER NOT NULL DEFAULT 0,
+            timestamp        TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX token_usage_user_ts_idx  ON token_usage (user_id, timestamp DESC);")
+    op.execute("CREATE INDEX token_usage_key_ts_idx   ON token_usage (api_key, timestamp DESC);")
+    op.execute("CREATE INDEX token_usage_agent_ts_idx ON token_usage (agent_id, timestamp DESC);")
+
+    op.execute("""
+        CREATE TABLE user_logs (
+            id        BIGSERIAL PRIMARY KEY,
+            user_id   TEXT,
+            endpoint  TEXT,
+            timestamp TIMESTAMPTZ NOT NULL DEFAULT now(),
+            data      JSONB
+        );
+    """)
+    op.execute("CREATE INDEX user_logs_user_ts_idx ON user_logs (user_id, timestamp DESC);")
+
+    op.execute("""
+        CREATE TABLE feedback (
+            id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            conversation_id UUID NOT NULL,              -- FK added later in this migration
+            user_id         TEXT NOT NULL,
+            question_index  INTEGER NOT NULL,
+            feedback_text   TEXT,
+            timestamp       TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX feedback_conv_idx ON feedback (conversation_id);")
+
+    # Append-only debug/error log. The Mongo doc has both `_id` (auto) and an
+    # `id` field (the activity id). Here the serial PK owns `id`; the
+    # application-level identifier is renamed to `activity_id`.
+    op.execute("""
+        CREATE TABLE stack_logs (
+            id          BIGSERIAL PRIMARY KEY,
+            activity_id TEXT NOT NULL,
+            endpoint    TEXT,
+            level       TEXT,
+            user_id     TEXT,
+            api_key     TEXT,
+            query       TEXT,
+            stacks      JSONB NOT NULL DEFAULT '[]'::jsonb,
+            timestamp   TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX stack_logs_timestamp_idx ON stack_logs (timestamp DESC);")
+    op.execute("CREATE INDEX stack_logs_user_ts_idx   ON stack_logs (user_id, timestamp DESC);")
+    op.execute("CREATE INDEX stack_logs_level_ts_idx  ON stack_logs (level, timestamp DESC);")
+    op.execute("CREATE INDEX stack_logs_activity_idx  ON stack_logs (activity_id);")
+
+    # ------------------------------------------------------------------
+    # Tier 2: FK-bearing tables
+    # ------------------------------------------------------------------
+    op.execute("""
+        CREATE TABLE agent_folders (
+            id          UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id     TEXT NOT NULL,
+            name        TEXT NOT NULL,
+            description TEXT,
+            created_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at  TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX agent_folders_user_idx ON agent_folders (user_id);")
+
+    op.execute("""
+        CREATE TABLE sources (
+            id         UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id    TEXT,                 -- NULL for system/template sources
+            name       TEXT NOT NULL,
+            type       TEXT,
+            metadata   JSONB NOT NULL DEFAULT '{}'::jsonb,
+            created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX sources_user_idx ON sources (user_id);")
+
+    op.execute("""
+        CREATE TABLE agents (
+            id                     UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id                TEXT NOT NULL,
+            name                   TEXT NOT NULL,
+            description            TEXT,
+            agent_type             TEXT,
+            status                 TEXT NOT NULL,
+            key                    CITEXT UNIQUE,
+            source_id              UUID REFERENCES sources(id) ON DELETE SET NULL,
+            extra_source_ids       UUID[] NOT NULL DEFAULT '{}',
+            chunks                 INTEGER,
+            retriever              TEXT,
+            prompt_id              UUID REFERENCES prompts(id) ON DELETE SET NULL,
+            tools                  JSONB NOT NULL DEFAULT '[]'::jsonb,
+            json_schema            JSONB,
+            models                 JSONB,
+            default_model_id       TEXT,
+            folder_id              UUID REFERENCES agent_folders(id) ON DELETE SET NULL,
+            limited_token_mode     BOOLEAN NOT NULL DEFAULT false,
+            token_limit            INTEGER,
+            limited_request_mode   BOOLEAN NOT NULL DEFAULT false,
+            request_limit          INTEGER,
+            shared                 BOOLEAN NOT NULL DEFAULT false,
+            incoming_webhook_token CITEXT UNIQUE,
+            created_at             TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at             TIMESTAMPTZ NOT NULL DEFAULT now(),
+            last_used_at           TIMESTAMPTZ
+        );
+    """)
+    op.execute("CREATE INDEX agents_user_idx   ON agents (user_id);")
+    op.execute("CREATE INDEX agents_shared_idx ON agents (shared) WHERE shared = true;")
+    op.execute("CREATE INDEX agents_status_idx ON agents (status);")
+
+    # Backfill the token_usage.agent_id FK now that agents exists.
+    op.execute("""
+        ALTER TABLE token_usage
+          ADD CONSTRAINT token_usage_agent_fk
+          FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL;
+    """)
+
+    op.execute("""
+        CREATE TABLE attachments (
+            id          UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id     TEXT NOT NULL,
+            filename    TEXT NOT NULL,
+            upload_path TEXT NOT NULL,
+            mime_type   TEXT,
+            size        BIGINT,
+            created_at  TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX attachments_user_idx ON attachments (user_id);")
+
+    op.execute("""
+        CREATE TABLE memories (
+            id         UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id    TEXT NOT NULL,
+            tool_id    UUID REFERENCES user_tools(id) ON DELETE CASCADE,
+            path       TEXT NOT NULL,
+            content    TEXT NOT NULL,
+            updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("""
+        CREATE UNIQUE INDEX memories_user_tool_path_uidx
+            ON memories (user_id, tool_id, path);
+    """)
+    op.execute("""
+        CREATE INDEX memories_path_prefix_idx
+            ON memories (user_id, tool_id, path text_pattern_ops);
+    """)
+
+    op.execute("""
+        CREATE TABLE todos (
+            id         UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id    TEXT NOT NULL,
+            tool_id    UUID REFERENCES user_tools(id) ON DELETE CASCADE,
+            title      TEXT NOT NULL,
+            completed  BOOLEAN NOT NULL DEFAULT false,
+            created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX todos_user_tool_idx ON todos (user_id, tool_id);")
+
+    op.execute("""
+        CREATE TABLE notes (
+            id         UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id    TEXT NOT NULL,
+            tool_id    UUID REFERENCES user_tools(id) ON DELETE CASCADE,
+            title      TEXT NOT NULL,
+            content    TEXT NOT NULL,
+            created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX notes_user_tool_idx ON notes (user_id, tool_id);")
+
+    op.execute("""
+        CREATE TABLE connector_sessions (
+            id           UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id      TEXT NOT NULL,
+            provider     TEXT NOT NULL,
+            session_data JSONB NOT NULL,
+            expires_at   TIMESTAMPTZ,
+            created_at   TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("""
+        CREATE INDEX connector_sessions_user_provider_idx
+            ON connector_sessions (user_id, provider);
+    """)
+    op.execute("""
+        CREATE INDEX connector_sessions_expiry_idx
+            ON connector_sessions (expires_at) WHERE expires_at IS NOT NULL;
+    """)
+
+    # ------------------------------------------------------------------
+    # Tier 3: conversations, pending_tool_state, workflows
+    # ------------------------------------------------------------------
+    op.execute("""
+        CREATE TABLE conversations (
+            id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id         TEXT NOT NULL,
+            agent_id        UUID REFERENCES agents(id) ON DELETE SET NULL,
+            name            TEXT,
+            api_key         TEXT,
+            is_shared_usage BOOLEAN NOT NULL DEFAULT false,
+            shared_token    TEXT,
+            date            TIMESTAMPTZ NOT NULL DEFAULT now(),
+            created_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at      TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX conversations_user_date_idx ON conversations (user_id, date DESC);")
+    op.execute("CREATE INDEX conversations_agent_idx     ON conversations (agent_id);")
+    op.execute("""
+        CREATE INDEX conversations_shared_token_idx
+            ON conversations (shared_token) WHERE shared_token IS NOT NULL;
+    """)
+
+    op.execute("""
+        CREATE TABLE conversation_messages (
+            id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            conversation_id UUID NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
+            position        INTEGER NOT NULL,
+            prompt          TEXT,
+            response        TEXT,
+            thought         TEXT,
+            sources         JSONB NOT NULL DEFAULT '[]'::jsonb,
+            tool_calls      JSONB NOT NULL DEFAULT '[]'::jsonb,
+            attachments     UUID[] NOT NULL DEFAULT '{}',
+            model_id        TEXT,
+            metadata        JSONB NOT NULL DEFAULT '{}'::jsonb,
+            feedback        JSONB,
+            timestamp       TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("""
+        CREATE UNIQUE INDEX conversation_messages_conv_pos_uidx
+            ON conversation_messages (conversation_id, position);
+    """)
+
+    # Backfill the feedback.conversation_id FK now that conversations exists.
+    op.execute("""
+        ALTER TABLE feedback
+          ADD CONSTRAINT feedback_conv_fk
+          FOREIGN KEY (conversation_id) REFERENCES conversations(id) ON DELETE CASCADE;
+    """)
+
+    op.execute("""
+        CREATE TABLE shared_conversations (
+            id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            conversation_id UUID NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
+            user_id         TEXT NOT NULL,
+            prompt_id       UUID REFERENCES prompts(id) ON DELETE SET NULL,
+            chunks          INTEGER,
+            is_promptable   BOOLEAN NOT NULL DEFAULT false,
+            created_at      TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX shared_conversations_user_idx ON shared_conversations (user_id);")
+    op.execute("CREATE INDEX shared_conversations_conv_idx ON shared_conversations (conversation_id);")
+
+    # Paused-tool continuation state. The Mongo version relies on a TTL index;
+    # Postgres has no native TTL, so a Celery beat task (added in Phase 3)
+    # deletes rows where expires_at < now() once a minute. The unique
+    # constraint on (conversation_id, user_id) matches the existing upsert
+    # semantics.
+    op.execute("""
+        CREATE TABLE pending_tool_state (
+            id                 UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            conversation_id    UUID NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
+            user_id            TEXT NOT NULL,
+            messages           JSONB NOT NULL,
+            pending_tool_calls JSONB NOT NULL,
+            tools_dict         JSONB NOT NULL,
+            tool_schemas       JSONB NOT NULL,
+            agent_config       JSONB NOT NULL,
+            client_tools       JSONB,
+            created_at         TIMESTAMPTZ NOT NULL DEFAULT now(),
+            expires_at         TIMESTAMPTZ NOT NULL
+        );
+    """)
+    op.execute("""
+        CREATE UNIQUE INDEX pending_tool_state_conv_user_uidx
+            ON pending_tool_state (conversation_id, user_id);
+    """)
+    op.execute("""
+        CREATE INDEX pending_tool_state_expires_idx
+            ON pending_tool_state (expires_at);
+    """)
+
+    # Workflows
+    op.execute("""
+        CREATE TABLE workflows (
+            id          UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            user_id     TEXT NOT NULL,
+            name        TEXT NOT NULL,
+            description TEXT,
+            created_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at  TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+    """)
+    op.execute("CREATE INDEX workflows_user_idx ON workflows (user_id);")
+
+    op.execute("""
+        CREATE TABLE workflow_nodes (
+            id            UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            workflow_id   UUID NOT NULL REFERENCES workflows(id) ON DELETE CASCADE,
+            graph_version INTEGER NOT NULL,
+            node_type     TEXT NOT NULL,
+            config        JSONB NOT NULL DEFAULT '{}'::jsonb
+        );
+    """)
+    op.execute("""
+        CREATE INDEX workflow_nodes_workflow_version_idx
+            ON workflow_nodes (workflow_id, graph_version);
+    """)
+
+    op.execute("""
+        CREATE TABLE workflow_edges (
+            id            UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            workflow_id   UUID NOT NULL REFERENCES workflows(id) ON DELETE CASCADE,
+            graph_version INTEGER NOT NULL,
+            from_node_id  UUID NOT NULL REFERENCES workflow_nodes(id) ON DELETE CASCADE,
+            to_node_id    UUID NOT NULL REFERENCES workflow_nodes(id) ON DELETE CASCADE,
+            config        JSONB NOT NULL DEFAULT '{}'::jsonb
+        );
+    """)
+    op.execute("""
+        CREATE INDEX workflow_edges_workflow_version_idx
+            ON workflow_edges (workflow_id, graph_version);
+    """)
+
+    op.execute("""
+        CREATE TABLE workflow_runs (
+            id          UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+            workflow_id UUID NOT NULL REFERENCES workflows(id) ON DELETE CASCADE,
+            user_id     TEXT NOT NULL,
+            status      TEXT NOT NULL,
+            started_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
+            ended_at    TIMESTAMPTZ,
+            result      JSONB
+        );
+    """)
+    op.execute("CREATE INDEX workflow_runs_workflow_idx ON workflow_runs (workflow_id);")
+    op.execute("CREATE INDEX workflow_runs_user_idx     ON workflow_runs (user_id);")
+
+
+def downgrade() -> None:
+    # Reverse dependency order. CASCADE would handle FKs anyway, but explicit
+    # is clearer for anyone reading the migration.
+    op.execute("DROP TABLE IF EXISTS workflow_runs CASCADE;")
+    op.execute("DROP TABLE IF EXISTS workflow_edges CASCADE;")
+    op.execute("DROP TABLE IF EXISTS workflow_nodes CASCADE;")
+    op.execute("DROP TABLE IF EXISTS workflows CASCADE;")
+    op.execute("DROP TABLE IF EXISTS pending_tool_state CASCADE;")
+    op.execute("DROP TABLE IF EXISTS shared_conversations CASCADE;")
+    op.execute("DROP TABLE IF EXISTS conversation_messages CASCADE;")
+    op.execute("DROP TABLE IF EXISTS conversations CASCADE;")
+    op.execute("DROP TABLE IF EXISTS connector_sessions CASCADE;")
+    op.execute("DROP TABLE IF EXISTS notes CASCADE;")
+    op.execute("DROP TABLE IF EXISTS todos CASCADE;")
+    op.execute("DROP TABLE IF EXISTS memories CASCADE;")
+    op.execute("DROP TABLE IF EXISTS attachments CASCADE;")
+    op.execute("DROP TABLE IF EXISTS agents CASCADE;")
+    op.execute("DROP TABLE IF EXISTS sources CASCADE;")
+    op.execute("DROP TABLE IF EXISTS agent_folders CASCADE;")
+    op.execute("DROP TABLE IF EXISTS stack_logs CASCADE;")
+    op.execute("DROP TABLE IF EXISTS feedback CASCADE;")
+    op.execute("DROP TABLE IF EXISTS user_logs CASCADE;")
+    op.execute("DROP TABLE IF EXISTS token_usage CASCADE;")
+    op.execute("DROP TABLE IF EXISTS user_tools CASCADE;")
+    op.execute("DROP TABLE IF EXISTS prompts CASCADE;")
+    op.execute("DROP TABLE IF EXISTS users CASCADE;")
+    # Extensions are intentionally left in place — they may be shared with
+    # pgvector or other extensions already enabled on the cluster.

application/api/answer/routes/answer.py

@@ -74,57 +74,76 @@ class AnswerResource(Resource, BaseAnswerResource):
         decoded_token = getattr(request, "decoded_token", None)
         processor = StreamProcessor(data, decoded_token)
         try:
-            agent = processor.build_agent(data.get("question", ""))
-            if not processor.decoded_token:
-                return make_response({"error": "Unauthorized"}, 401)
+            # ---- Continuation mode ----
+            if data.get("tool_actions"):
+                (
+                    agent,
+                    messages,
+                    tools_dict,
+                    pending_tool_calls,
+                    tool_actions,
+                ) = processor.resume_from_tool_actions(
+                    data["tool_actions"], data["conversation_id"]
+                )
+                if not processor.decoded_token:
+                    return make_response({"error": "Unauthorized"}, 401)
+                if error := self.check_usage(processor.agent_config):
+                    return error
+                stream = self.complete_stream(
+                    question="",
+                    agent=agent,
+                    conversation_id=processor.conversation_id,
+                    user_api_key=processor.agent_config.get("user_api_key"),
+                    decoded_token=processor.decoded_token,
+                    agent_id=processor.agent_id,
+                    model_id=processor.model_id,
+                    _continuation={
+                        "messages": messages,
+                        "tools_dict": tools_dict,
+                        "pending_tool_calls": pending_tool_calls,
+                        "tool_actions": tool_actions,
+                    },
+                )
+            else:
+                # ---- Normal mode ----
+                agent = processor.build_agent(data.get("question", ""))
+                if not processor.decoded_token:
+                    return make_response({"error": "Unauthorized"}, 401)
 
-            if error := self.check_usage(processor.agent_config):
-                return error
+                if error := self.check_usage(processor.agent_config):
+                    return error
+
+                stream = self.complete_stream(
+                    question=data["question"],
+                    agent=agent,
+                    conversation_id=processor.conversation_id,
+                    user_api_key=processor.agent_config.get("user_api_key"),
+                    decoded_token=processor.decoded_token,
+                    isNoneDoc=data.get("isNoneDoc"),
+                    index=None,
+                    should_save_conversation=data.get("save_conversation", True),
+                    agent_id=processor.agent_id,
+                    is_shared_usage=processor.is_shared_usage,
+                    shared_token=processor.shared_token,
+                    model_id=processor.model_id,
+                )
 
-            stream = self.complete_stream(
-                question=data["question"],
-                agent=agent,
-                conversation_id=processor.conversation_id,
-                user_api_key=processor.agent_config.get("user_api_key"),
-                decoded_token=processor.decoded_token,
-                isNoneDoc=data.get("isNoneDoc"),
-                index=None,
-                should_save_conversation=data.get("save_conversation", True),
-                agent_id=processor.agent_id,
-                is_shared_usage=processor.is_shared_usage,
-                shared_token=processor.shared_token,
-                model_id=processor.model_id,
-            )
             stream_result = self.process_response_stream(stream)
 
-            if len(stream_result) == 7:
-                (
-                    conversation_id,
-                    response,
-                    sources,
-                    tool_calls,
-                    thought,
-                    error,
-                    structured_info,
-                ) = stream_result
-            else:
-                conversation_id, response, sources, tool_calls, thought, error = (
-                    stream_result
-                )
-                structured_info = None
+            if stream_result["error"]:
+                return make_response({"error": stream_result["error"]}, 400)
 
-            if error:
-                return make_response({"error": error}, 400)
             result = {
-                "conversation_id": conversation_id,
-                "answer": response,
-                "sources": sources,
-                "tool_calls": tool_calls,
-                "thought": thought,
+                "conversation_id": stream_result["conversation_id"],
+                "answer": stream_result["answer"],
+                "sources": stream_result["sources"],
+                "tool_calls": stream_result["tool_calls"],
+                "thought": stream_result["thought"],
             }
 
-            if structured_info:
-                result.update(structured_info)
+            extra_info = stream_result.get("extra")
+            if extra_info:
+                result.update(extra_info)
         except Exception as e:
             logger.error(
                 f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}",

application/api/answer/routes/base.py

@@ -6,6 +6,7 @@ from typing import Any, Dict, Generator, List, Optional
 from flask import jsonify, make_response, Response
 from flask_restx import Namespace
 
+from application.api.answer.services.continuation_service import ContinuationService
 from application.api.answer.services.conversation_service import ConversationService
 from application.core.model_utils import (
     get_api_key_for_provider,
@@ -39,7 +40,16 @@ class BaseAnswerResource:
     def validate_request(
         self, data: Dict[str, Any], require_conversation_id: bool = False
     ) -> Optional[Response]:
-        """Common request validation"""
+        """Common request validation.
+
+        Continuation requests (``tool_actions`` present) require
+        ``conversation_id`` but not ``question``.
+        """
+        if data.get("tool_actions"):
+            # Continuation mode — question is not required
+            if missing := check_required_fields(data, ["conversation_id"]):
+                return missing
+            return None
         required_fields = ["question"]
         if require_conversation_id:
             required_fields.append("conversation_id")
@@ -177,6 +187,7 @@ class BaseAnswerResource:
         is_shared_usage: bool = False,
         shared_token: Optional[str] = None,
         model_id: Optional[str] = None,
+        _continuation: Optional[Dict] = None,
     ) -> Generator[str, None, None]:
         """
         Generator function that streams the complete conversation response.
@@ -207,8 +218,19 @@ class BaseAnswerResource:
             schema_info = None
             structured_chunks = []
             query_metadata = {}
+            paused = False
 
-            for line in agent.gen(query=question):
+            if _continuation:
+                gen_iter = agent.gen_continuation(
+                    messages=_continuation["messages"],
+                    tools_dict=_continuation["tools_dict"],
+                    pending_tool_calls=_continuation["pending_tool_calls"],
+                    tool_actions=_continuation["tool_actions"],
+                )
+            else:
+                gen_iter = agent.gen(query=question)
+
+            for line in gen_iter:
                 if "metadata" in line:
                     query_metadata.update(line["metadata"])
                 elif "answer" in line:
@@ -244,15 +266,21 @@ class BaseAnswerResource:
                     data = json.dumps({"type": "thought", "thought": line["thought"]})
                     yield f"data: {data}\n\n"
                 elif "type" in line:
-                    if line.get("type") == "error":
+                    if line.get("type") == "tool_calls_pending":
+                        # Save continuation state and end the stream
+                        paused = True
+                        data = json.dumps(line)
+                        yield f"data: {data}\n\n"
+                    elif line.get("type") == "error":
                         sanitized_error = {
                             "type": "error",
                             "error": sanitize_api_error(line.get("error", "An error occurred"))
                         }
                         data = json.dumps(sanitized_error)
+                        yield f"data: {data}\n\n"
                     else:
                         data = json.dumps(line)
-                    yield f"data: {data}\n\n"
+                        yield f"data: {data}\n\n"
             if is_structured and structured_chunks:
                 structured_data = {
                     "type": "structured_answer",
@@ -262,6 +290,93 @@ class BaseAnswerResource:
                 }
                 data = json.dumps(structured_data)
                 yield f"data: {data}\n\n"
+
+            # ---- Paused: save continuation state and end stream early ----
+            if paused:
+                continuation = getattr(agent, "_pending_continuation", None)
+                if continuation:
+                    # Ensure we have a conversation_id — create a partial
+                    # conversation if this is the first turn.
+                    if not conversation_id and should_save_conversation:
+                        try:
+                            provider = (
+                                get_provider_from_model_id(model_id)
+                                if model_id
+                                else settings.LLM_PROVIDER
+                            )
+                            sys_api_key = get_api_key_for_provider(
+                                provider or settings.LLM_PROVIDER
+                            )
+                            llm = LLMCreator.create_llm(
+                                provider or settings.LLM_PROVIDER,
+                                api_key=sys_api_key,
+                                user_api_key=user_api_key,
+                                decoded_token=decoded_token,
+                                model_id=model_id,
+                                agent_id=agent_id,
+                            )
+                            conversation_id = (
+                                self.conversation_service.save_conversation(
+                                    None,
+                                    question,
+                                    response_full,
+                                    thought,
+                                    source_log_docs,
+                                    tool_calls,
+                                    llm,
+                                    model_id or self.default_model_id,
+                                    decoded_token,
+                                    api_key=user_api_key,
+                                    agent_id=agent_id,
+                                    is_shared_usage=is_shared_usage,
+                                    shared_token=shared_token,
+                                )
+                            )
+                        except Exception as e:
+                            logger.error(
+                                f"Failed to create conversation for continuation: {e}",
+                                exc_info=True,
+                            )
+
+                    if conversation_id:
+                        try:
+                            cont_service = ContinuationService()
+                            cont_service.save_state(
+                                conversation_id=str(conversation_id),
+                                user=decoded_token.get("sub", "local"),
+                                messages=continuation["messages"],
+                                pending_tool_calls=continuation["pending_tool_calls"],
+                                tools_dict=continuation["tools_dict"],
+                                tool_schemas=getattr(agent, "tools", []),
+                                agent_config={
+                                    "model_id": model_id or self.default_model_id,
+                                    "llm_name": getattr(agent, "llm_name", settings.LLM_PROVIDER),
+                                    "api_key": getattr(agent, "api_key", None),
+                                    "user_api_key": user_api_key,
+                                    "agent_id": agent_id,
+                                    "agent_type": agent.__class__.__name__,
+                                    "prompt": getattr(agent, "prompt", ""),
+                                    "json_schema": getattr(agent, "json_schema", None),
+                                    "retriever_config": getattr(agent, "retriever_config", None),
+                                },
+                                client_tools=getattr(
+                                    agent.tool_executor, "client_tools", None
+                                ),
+                            )
+                        except Exception as e:
+                            logger.error(
+                                f"Failed to save continuation state: {str(e)}",
+                                exc_info=True,
+                            )
+
+                id_data = {"type": "id", "id": str(conversation_id)}
+                data = json.dumps(id_data)
+                yield f"data: {data}\n\n"
+
+                data = json.dumps({"type": "end"})
+                yield f"data: {data}\n\n"
+                return
+
             if isNoneDoc:
                 for doc in source_log_docs:
                     doc["source"] = "None"
@@ -425,8 +540,13 @@ class BaseAnswerResource:
             yield f"data: {data}\n\n"
             return
 
-    def process_response_stream(self, stream):
-        """Process the stream response for non-streaming endpoint"""
+    def process_response_stream(self, stream) -> Dict[str, Any]:
+        """Process the stream response for non-streaming endpoint.
+
+        Returns:
+            Dict with keys: conversation_id, answer, sources, tool_calls,
+            thought, error, and optional extra.
+        """
         conversation_id = ""
         response_full = ""
         source_log_docs = []
@@ -435,6 +555,7 @@ class BaseAnswerResource:
         stream_ended = False
         is_structured = False
         schema_info = None
+        pending_tool_calls = None
 
         for line in stream:
             try:
@@ -453,11 +574,22 @@ class BaseAnswerResource:
                     source_log_docs = event["source"]
                 elif event["type"] == "tool_calls":
                     tool_calls = event["tool_calls"]
+                elif event["type"] == "tool_calls_pending":
+                    pending_tool_calls = event.get("data", {}).get(
+                        "pending_tool_calls", []
+                    )
                 elif event["type"] == "thought":
                     thought = event["thought"]
                 elif event["type"] == "error":
                     logger.error(f"Error from stream: {event['error']}")
-                    return None, None, None, None, event["error"], None
+                    return {
+                        "conversation_id": None,
+                        "answer": None,
+                        "sources": None,
+                        "tool_calls": None,
+                        "thought": None,
+                        "error": event["error"],
+                    }
                 elif event["type"] == "end":
                     stream_ended = True
             except (json.JSONDecodeError, KeyError) as e:
@@ -465,18 +597,30 @@ class BaseAnswerResource:
                 continue
         if not stream_ended:
             logger.error("Stream ended unexpectedly without an 'end' event.")
-            return None, None, None, None, "Stream ended unexpectedly", None
-        result = (
-            conversation_id,
-            response_full,
-            source_log_docs,
-            tool_calls,
-            thought,
-            None,
-        )
+            return {
+                "conversation_id": None,
+                "answer": None,
+                "sources": None,
+                "tool_calls": None,
+                "thought": None,
+                "error": "Stream ended unexpectedly",
+            }
+
+        result: Dict[str, Any] = {
+            "conversation_id": conversation_id,
+            "answer": response_full,
+            "sources": source_log_docs,
+            "tool_calls": tool_calls,
+            "thought": thought,
+            "error": None,
+        }
+
+        if pending_tool_calls is not None:
+            result["extra"] = {"pending_tool_calls": pending_tool_calls}
 
         if is_structured:
-            result = result + ({"structured": True, "schema": schema_info},)
+            result["extra"] = {"structured": True, "schema": schema_info}
+
         return result
 
     def error_stream_generate(self, err_response):

application/api/answer/routes/stream.py

@@ -79,7 +79,47 @@ class StreamResource(Resource, BaseAnswerResource):
             return error
         decoded_token = getattr(request, "decoded_token", None)
         processor = StreamProcessor(data, decoded_token)
+
         try:
+            # ---- Continuation mode ----
+            if data.get("tool_actions"):
+                (
+                    agent,
+                    messages,
+                    tools_dict,
+                    pending_tool_calls,
+                    tool_actions,
+                ) = processor.resume_from_tool_actions(
+                    data["tool_actions"], data["conversation_id"]
+                )
+                if not processor.decoded_token:
+                    return Response(
+                        self.error_stream_generate("Unauthorized"),
+                        status=401,
+                        mimetype="text/event-stream",
+                    )
+                if error := self.check_usage(processor.agent_config):
+                    return error
+                return Response(
+                    self.complete_stream(
+                        question="",
+                        agent=agent,
+                        conversation_id=processor.conversation_id,
+                        user_api_key=processor.agent_config.get("user_api_key"),
+                        decoded_token=processor.decoded_token,
+                        agent_id=processor.agent_id,
+                        model_id=processor.model_id,
+                        _continuation={
+                            "messages": messages,
+                            "tools_dict": tools_dict,
+                            "pending_tool_calls": pending_tool_calls,
+                            "tool_actions": tool_actions,
+                        },
+                    ),
+                    mimetype="text/event-stream",
+                )
+
+            # ---- Normal mode ----
             agent = processor.build_agent(data["question"])
             if not processor.decoded_token:
                 return Response(

application/api/answer/services/compression/message_builder.py

@@ -1,5 +1,6 @@
 """Message reconstruction utilities for compression."""
 
+import json
 import logging
 import uuid
 from typing import Dict, List, Optional
@@ -49,28 +50,35 @@ class MessageBuilder:
             if include_tool_calls and "tool_calls" in query:
                 for tool_call in query["tool_calls"]:
                     call_id = tool_call.get("call_id") or str(uuid.uuid4())
-
-                    function_call_dict = {
-                        "function_call": {
-                            "name": tool_call.get("action_name"),
-                            "args": tool_call.get("arguments"),
-                            "call_id": call_id,
-                        }
-                    }
-                    function_response_dict = {
-                        "function_response": {
-                            "name": tool_call.get("action_name"),
-                            "response": {"result": tool_call.get("result")},
-                            "call_id": call_id,
-                        }
-                    }
-
-                    messages.append(
-                        {"role": "assistant", "content": [function_call_dict]}
+                    args = tool_call.get("arguments")
+                    args_str = (
+                        json.dumps(args)
+                        if isinstance(args, dict)
+                        else (args or "{}")
                     )
-                    messages.append(
-                        {"role": "tool", "content": [function_response_dict]}
+                    messages.append({
+                        "role": "assistant",
+                        "content": None,
+                        "tool_calls": [{
+                            "id": call_id,
+                            "type": "function",
+                            "function": {
+                                "name": tool_call.get("action_name", ""),
+                                "arguments": args_str,
+                            },
+                        }],
+                    })
+                    result = tool_call.get("result")
+                    result_str = (
+                        json.dumps(result)
+                        if not isinstance(result, str)
+                        else (result or "")
                     )
+                    messages.append({
+                        "role": "tool",
+                        "tool_call_id": call_id,
+                        "content": result_str,
+                    })
 
         # If no recent queries (everything was compressed), add a continuation user message
         if len(recent_queries) == 0 and compressed_summary:
@@ -180,28 +188,35 @@ class MessageBuilder:
             if include_tool_calls and "tool_calls" in query:
                 for tool_call in query["tool_calls"]:
                     call_id = tool_call.get("call_id") or str(uuid.uuid4())
-
-                    function_call_dict = {
-                        "function_call": {
-                            "name": tool_call.get("action_name"),
-                            "args": tool_call.get("arguments"),
-                            "call_id": call_id,
-                        }
-                    }
-                    function_response_dict = {
-                        "function_response": {
-                            "name": tool_call.get("action_name"),
-                            "response": {"result": tool_call.get("result")},
-                            "call_id": call_id,
-                        }
-                    }
-
-                    rebuilt_messages.append(
-                        {"role": "assistant", "content": [function_call_dict]}
+                    args = tool_call.get("arguments")
+                    args_str = (
+                        json.dumps(args)
+                        if isinstance(args, dict)
+                        else (args or "{}")
                     )
-                    rebuilt_messages.append(
-                        {"role": "tool", "content": [function_response_dict]}
+                    rebuilt_messages.append({
+                        "role": "assistant",
+                        "content": None,
+                        "tool_calls": [{
+                            "id": call_id,
+                            "type": "function",
+                            "function": {
+                                "name": tool_call.get("action_name", ""),
+                                "arguments": args_str,
+                            },
+                        }],
+                    })
+                    result = tool_call.get("result")
+                    result_str = (
+                        json.dumps(result)
+                        if not isinstance(result, str)
+                        else (result or "")
                     )
+                    rebuilt_messages.append({
+                        "role": "tool",
+                        "tool_call_id": call_id,
+                        "content": result_str,
+                    })
 
         # If no recent queries (everything was compressed), add a continuation user message
         if len(recent_queries) == 0 and compressed_summary:

application/api/answer/services/continuation_service.py

@@ -0,0 +1,141 @@
+"""Service for saving and restoring tool-call continuation state.
+
+When a stream pauses (tool needs approval or client-side execution),
+the full execution state is persisted to MongoDB so the client can
+resume later by sending tool_actions.
+"""
+
+import datetime
+import logging
+from typing import Any, Dict, List, Optional
+
+from bson import ObjectId
+
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+# TTL for pending states — auto-cleaned after this period
+PENDING_STATE_TTL_SECONDS = 30 * 60  # 30 minutes
+
+
+def _make_serializable(obj: Any) -> Any:
+    """Recursively convert MongoDB ObjectIds and other non-JSON types."""
+    if isinstance(obj, ObjectId):
+        return str(obj)
+    if isinstance(obj, dict):
+        return {str(k): _make_serializable(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_make_serializable(v) for v in obj]
+    if isinstance(obj, bytes):
+        return obj.decode("utf-8", errors="replace")
+    return obj
+
+
+class ContinuationService:
+    """Manages pending tool-call state in MongoDB."""
+
+    def __init__(self):
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        self.collection = db["pending_tool_state"]
+        self._ensure_indexes()
+
+    def _ensure_indexes(self):
+        try:
+            self.collection.create_index(
+                "expires_at", expireAfterSeconds=0
+            )
+            self.collection.create_index(
+                [("conversation_id", 1), ("user", 1)], unique=True
+            )
+        except Exception:
+            # Indexes may already exist or mongomock doesn't support TTL
+            pass
+
+    def save_state(
+        self,
+        conversation_id: str,
+        user: str,
+        messages: List[Dict],
+        pending_tool_calls: List[Dict],
+        tools_dict: Dict,
+        tool_schemas: List[Dict],
+        agent_config: Dict,
+        client_tools: Optional[List[Dict]] = None,
+    ) -> str:
+        """Save execution state for later continuation.
+
+        Args:
+            conversation_id: The conversation this state belongs to.
+            user: Owner user ID.
+            messages: Full messages array at the pause point.
+            pending_tool_calls: Tool calls awaiting client action.
+            tools_dict: Serializable tools configuration dict.
+            tool_schemas: LLM-formatted tool schemas (agent.tools).
+            agent_config: Config needed to recreate the agent on resume.
+            client_tools: Client-provided tool schemas for client-side execution.
+
+        Returns:
+            The string ID of the saved state document.
+        """
+        now = datetime.datetime.now(datetime.timezone.utc)
+        expires_at = now + datetime.timedelta(seconds=PENDING_STATE_TTL_SECONDS)
+
+        doc = {
+            "conversation_id": conversation_id,
+            "user": user,
+            "messages": _make_serializable(messages),
+            "pending_tool_calls": _make_serializable(pending_tool_calls),
+            "tools_dict": _make_serializable(tools_dict),
+            "tool_schemas": _make_serializable(tool_schemas),
+            "agent_config": _make_serializable(agent_config),
+            "client_tools": _make_serializable(client_tools) if client_tools else None,
+            "created_at": now,
+            "expires_at": expires_at,
+        }
+
+        # Upsert — only one pending state per conversation per user
+        result = self.collection.replace_one(
+            {"conversation_id": conversation_id, "user": user},
+            doc,
+            upsert=True,
+        )
+        state_id = str(result.upserted_id) if result.upserted_id else conversation_id
+        logger.info(
+            f"Saved continuation state for conversation {conversation_id} "
+            f"with {len(pending_tool_calls)} pending tool call(s)"
+        )
+        return state_id
+
+    def load_state(
+        self, conversation_id: str, user: str
+    ) -> Optional[Dict[str, Any]]:
+        """Load pending continuation state.
+
+        Returns:
+            The state dict, or None if no pending state exists.
+        """
+        doc = self.collection.find_one(
+            {"conversation_id": conversation_id, "user": user}
+        )
+        if not doc:
+            return None
+        doc["_id"] = str(doc["_id"])
+        return doc
+
+    def delete_state(self, conversation_id: str, user: str) -> bool:
+        """Delete pending state after successful resumption.
+
+        Returns:
+            True if a document was deleted.
+        """
+        result = self.collection.delete_one(
+            {"conversation_id": conversation_id, "user": user}
+        )
+        if result.deleted_count:
+            logger.info(
+                f"Deleted continuation state for conversation {conversation_id}"
+            )
+        return result.deleted_count > 0

application/api/answer/services/stream_processor.py

@@ -112,6 +112,7 @@ class StreamProcessor:
         self._required_tool_actions: Optional[Dict[str, Set[Optional[str]]]] = None
         self.compressed_summary: Optional[str] = None
         self.compressed_summary_tokens: int = 0
+        self._agent_data: Optional[Dict[str, Any]] = None
 
     def initialize(self):
         """Initialize all required components for processing"""
@@ -359,22 +360,29 @@ class StreamProcessor:
         return data
 
     def _configure_source(self):
-        """Configure the source based on agent data"""
-        api_key = self.data.get("api_key") or self.agent_key
+        """Configure the source based on agent data.
 
-        if api_key:
-            agent_data = self._get_data_from_api_key(api_key)
+        The literal string ``"default"`` is a placeholder meaning "no
+        ingested source" and is normalized to an empty source so that no
+        retrieval is attempted.
+        """
+        if self._agent_data:
+            agent_data = self._agent_data
 
             if agent_data.get("sources") and len(agent_data["sources"]) > 0:
                 source_ids = [
-                    source["id"] for source in agent_data["sources"] if source.get("id")
+                    source["id"]
+                    for source in agent_data["sources"]
+                    if source.get("id") and source["id"] != "default"
                 ]
                 if source_ids:
                     self.source = {"active_docs": source_ids}
                 else:
                     self.source = {}
-                self.all_sources = agent_data["sources"]
-            elif agent_data.get("source"):
+                self.all_sources = [
+                    s for s in agent_data["sources"] if s.get("id") != "default"
+                ]
+            elif agent_data.get("source") and agent_data["source"] != "default":
                 self.source = {"active_docs": agent_data["source"]}
                 self.all_sources = [
                     {
@@ -387,11 +395,24 @@ class StreamProcessor:
                 self.all_sources = []
             return
         if "active_docs" in self.data:
-            self.source = {"active_docs": self.data["active_docs"]}
+            active_docs = self.data["active_docs"]
+            if active_docs and active_docs != "default":
+                self.source = {"active_docs": active_docs}
+            else:
+                self.source = {}
             return
         self.source = {}
         self.all_sources = []
 
+    def _has_active_docs(self) -> bool:
+        """Return True if a real document source is configured for retrieval."""
+        active_docs = self.source.get("active_docs") if self.source else None
+        if not active_docs:
+            return False
+        if active_docs == "default":
+            return False
+        return True
+
     def _resolve_agent_id(self) -> Optional[str]:
         """Resolve agent_id from request, then fall back to conversation context."""
         request_agent_id = self.data.get("agent_id")
@@ -433,48 +454,39 @@ class StreamProcessor:
         effective_key = self.data.get("api_key") or self.agent_key
 
         if effective_key:
-            data_key = self._get_data_from_api_key(effective_key)
-            if data_key.get("_id"):
-                self.agent_id = str(data_key.get("_id"))
+            self._agent_data = self._get_data_from_api_key(effective_key)
+            if self._agent_data.get("_id"):
+                self.agent_id = str(self._agent_data.get("_id"))
 
             self.agent_config.update(
                 {
-                    "prompt_id": data_key.get("prompt_id", "default"),
-                    "agent_type": data_key.get("agent_type", settings.AGENT_NAME),
+                    "prompt_id": self._agent_data.get("prompt_id", "default"),
+                    "agent_type": self._agent_data.get("agent_type", settings.AGENT_NAME),
                     "user_api_key": effective_key,
-                    "json_schema": data_key.get("json_schema"),
-                    "default_model_id": data_key.get("default_model_id", ""),
-                    "models": data_key.get("models", []),
+                    "json_schema": self._agent_data.get("json_schema"),
+                    "default_model_id": self._agent_data.get("default_model_id", ""),
+                    "models": self._agent_data.get("models", []),
+                    "allow_system_prompt_override": self._agent_data.get(
+                        "allow_system_prompt_override", False
+                    ),
                 }
             )
 
             # Set identity context
             if self.data.get("api_key"):
                 # External API key: use the key owner's identity
-                self.initial_user_id = data_key.get("user")
-                self.decoded_token = {"sub": data_key.get("user")}
+                self.initial_user_id = self._agent_data.get("user")
+                self.decoded_token = {"sub": self._agent_data.get("user")}
             elif self.is_shared_usage:
                 # Shared agent: keep the caller's identity
                 pass
             else:
                 # Owner using their own agent
-                self.decoded_token = {"sub": data_key.get("user")}
+                self.decoded_token = {"sub": self._agent_data.get("user")}
 
-            if data_key.get("source"):
-                self.source = {"active_docs": data_key["source"]}
-            if data_key.get("workflow"):
-                self.agent_config["workflow"] = data_key["workflow"]
-                self.agent_config["workflow_owner"] = data_key.get("user")
-            if data_key.get("retriever"):
-                self.retriever_config["retriever_name"] = data_key["retriever"]
-            if data_key.get("chunks") is not None:
-                try:
-                    self.retriever_config["chunks"] = int(data_key["chunks"])
-                except (ValueError, TypeError):
-                    logger.warning(
-                        f"Invalid chunks value: {data_key['chunks']}, using default value 2"
-                    )
-                    self.retriever_config["chunks"] = 2
+            if self._agent_data.get("workflow"):
+                self.agent_config["workflow"] = self._agent_data["workflow"]
+                self.agent_config["workflow_owner"] = self._agent_data.get("user")
         else:
             # No API key — default/workflow configuration
             agent_type = settings.AGENT_NAME
@@ -497,14 +509,45 @@ class StreamProcessor:
             )
 
     def _configure_retriever(self):
+        """Assemble retriever config with precedence: request > agent > default."""
         doc_token_limit = calculate_doc_token_budget(model_id=self.model_id)
 
+        # Start with defaults
+        retriever_name = "classic"
+        chunks = 2
+
+        # Layer agent-level config (if present)
+        if self._agent_data:
+            if self._agent_data.get("retriever"):
+                retriever_name = self._agent_data["retriever"]
+            if self._agent_data.get("chunks") is not None:
+                try:
+                    chunks = int(self._agent_data["chunks"])
+                except (ValueError, TypeError):
+                    logger.warning(
+                        f"Invalid agent chunks value: {self._agent_data['chunks']}, "
+                        "using default value 2"
+                    )
+
+        # Explicit request values win over agent config
+        if "retriever" in self.data:
+            retriever_name = self.data["retriever"]
+        if "chunks" in self.data:
+            try:
+                chunks = int(self.data["chunks"])
+            except (ValueError, TypeError):
+                logger.warning(
+                    f"Invalid request chunks value: {self.data['chunks']}, "
+                    "using default value 2"
+                )
+
         self.retriever_config = {
-            "retriever_name": self.data.get("retriever", "classic"),
-            "chunks": int(self.data.get("chunks", 2)),
+            "retriever_name": retriever_name,
+            "chunks": chunks,
             "doc_token_limit": doc_token_limit,
         }
 
+        # isNoneDoc without an API key forces no retrieval
         api_key = self.data.get("api_key") or self.agent_key
         if not api_key and "isNoneDoc" in self.data and self.data["isNoneDoc"]:
             self.retriever_config["chunks"] = 0
@@ -528,6 +571,9 @@ class StreamProcessor:
         if self.data.get("isNoneDoc", False) and not self.agent_id:
             logger.info("Pre-fetch skipped: isNoneDoc=True")
             return None, None
+        if not self._has_active_docs():
+            logger.info("Pre-fetch skipped: no active docs configured")
+            return None, None
         try:
             retriever = self.create_retriever()
             logger.info(
@@ -771,6 +817,121 @@ class StreamProcessor:
             logger.warning(f"Failed to fetch memory tool data: {str(e)}")
             return None
 
+    def resume_from_tool_actions(
+        self,
+        tool_actions: list,
+        conversation_id: str,
+    ):
+        """Resume a paused agent from saved continuation state.
+
+        Loads the pending state from MongoDB, recreates the agent with
+        the saved configuration, and returns an agent ready to call
+        ``gen_continuation()``.
+
+        Args:
+            tool_actions: Client-provided actions (approvals / results).
+            conversation_id: The conversation being resumed.
+
+        Returns:
+            Tuple of (agent, messages, tools_dict, pending_tool_calls, tool_actions).
+        """
+        from application.api.answer.services.continuation_service import (
+            ContinuationService,
+        )
+        from application.agents.agent_creator import AgentCreator
+        from application.agents.tool_executor import ToolExecutor
+        from application.llm.handlers.handler_creator import LLMHandlerCreator
+        from application.llm.llm_creator import LLMCreator
+
+        cont_service = ContinuationService()
+        state = cont_service.load_state(conversation_id, self.initial_user_id)
+        if not state:
+            raise ValueError("No pending tool state found for this conversation")
+
+        messages = state["messages"]
+        pending_tool_calls = state["pending_tool_calls"]
+        tools_dict = state["tools_dict"]
+        tool_schemas = state.get("tool_schemas", [])
+        agent_config = state["agent_config"]
+
+        model_id = agent_config.get("model_id")
+        llm_name = agent_config.get("llm_name", settings.LLM_PROVIDER)
+        api_key = agent_config.get("api_key")
+        user_api_key = agent_config.get("user_api_key")
+        agent_id = agent_config.get("agent_id")
+        prompt = agent_config.get("prompt", "")
+        json_schema = agent_config.get("json_schema")
+        retriever_config = agent_config.get("retriever_config")
+
+        # Recreate dependencies
+        system_api_key = api_key or get_api_key_for_provider(llm_name)
+        llm = LLMCreator.create_llm(
+            llm_name,
+            api_key=system_api_key,
+            user_api_key=user_api_key,
+            decoded_token=self.decoded_token,
+            model_id=model_id,
+            agent_id=agent_id,
+        )
+        llm_handler = LLMHandlerCreator.create_handler(llm_name or "default")
+        tool_executor = ToolExecutor(
+            user_api_key=user_api_key,
+            user=self.initial_user_id,
+            decoded_token=self.decoded_token,
+        )
+        tool_executor.conversation_id = conversation_id
+        # Restore client tools so they stay available for subsequent LLM calls
+        saved_client_tools = state.get("client_tools")
+        if saved_client_tools:
+            tool_executor.client_tools = saved_client_tools
+            # Re-merge into tools_dict (they may have been stripped during serialization)
+            tool_executor.merge_client_tools(tools_dict, saved_client_tools)
+
+        agent_type = agent_config.get("agent_type", "ClassicAgent")
+        # Map class names back to agent creator keys
+        type_map = {
+            "ClassicAgent": "classic",
+            "AgenticAgent": "agentic",
+            "ResearchAgent": "research",
+            "WorkflowAgent": "workflow",
+        }
+        agent_key = type_map.get(agent_type, "classic")
+
+        agent_kwargs = {
+            "endpoint": "stream",
+            "llm_name": llm_name,
+            "model_id": model_id,
+            "api_key": system_api_key,
+            "agent_id": agent_id,
+            "user_api_key": user_api_key,
+            "prompt": prompt,
+            "chat_history": [],
+            "decoded_token": self.decoded_token,
+            "json_schema": json_schema,
+            "llm": llm,
+            "llm_handler": llm_handler,
+            "tool_executor": tool_executor,
+        }
+
+        if agent_key in ("agentic", "research") and retriever_config:
+            agent_kwargs["retriever_config"] = retriever_config
+
+        agent = AgentCreator.create_agent(agent_key, **agent_kwargs)
+        agent.conversation_id = conversation_id
+        agent.initial_user_id = self.initial_user_id
+        agent.tools = tool_schemas
+
+        # Store config for the route layer
+        self.model_id = model_id
+        self.agent_id = agent_id
+        self.agent_config["user_api_key"] = user_api_key
+        self.conversation_id = conversation_id
+
+        # Delete state so it can't be replayed
+        cont_service.delete_state(conversation_id, self.initial_user_id)
+
+        return agent, messages, tools_dict, pending_tool_calls, tool_actions
+
     def create_agent(
         self,
         docs_together: Optional[str] = None,
@@ -795,15 +956,23 @@ class StreamProcessor:
                 raw_prompt = get_prompt(prompt_id, self.prompts_collection)
             self._prompt_content = raw_prompt
 
-        rendered_prompt = self.prompt_renderer.render_prompt(
-            prompt_content=raw_prompt,
-            user_id=self.initial_user_id,
-            request_id=self.data.get("request_id"),
-            passthrough_data=self.data.get("passthrough"),
-            docs=docs,
-            docs_together=docs_together,
-            tools_data=tools_data,
-        )
+        # Allow API callers to override the system prompt when the agent
+        # has opted in via allow_system_prompt_override.
+        if (
+            self.agent_config.get("allow_system_prompt_override", False)
+            and self.data.get("system_prompt_override")
+        ):
+            rendered_prompt = self.data["system_prompt_override"]
+        else:
+            rendered_prompt = self.prompt_renderer.render_prompt(
+                prompt_content=raw_prompt,
+                user_id=self.initial_user_id,
+                request_id=self.data.get("request_id"),
+                passthrough_data=self.data.get("passthrough"),
+                docs=docs,
+                docs_together=docs_together,
+                tools_data=tools_data,
+            )
 
         provider = (
             get_provider_from_model_id(self.model_id)
@@ -841,6 +1010,10 @@ class StreamProcessor:
             decoded_token=self.decoded_token,
         )
         tool_executor.conversation_id = self.conversation_id
+        # Pass client-side tools so they get merged in get_tools()
+        client_tools = self.data.get("client_tools")
+        if client_tools:
+            tool_executor.client_tools = client_tools
 
         # Base agent kwargs
         agent_kwargs = {

application/api/internal/routes.py

@@ -26,12 +26,20 @@ internal = Blueprint("internal", __name__)
 
 @internal.before_request
 def verify_internal_key():
-    """Verify INTERNAL_KEY for all internal endpoint requests."""
-    if settings.INTERNAL_KEY:
-        internal_key = request.headers.get("X-Internal-Key")
-        if not internal_key or internal_key != settings.INTERNAL_KEY:
-            logger.warning(f"Unauthorized internal API access attempt from {request.remote_addr}")
-            return jsonify({"error": "Unauthorized", "message": "Invalid or missing internal key"}), 401
+    """Verify INTERNAL_KEY for all internal endpoint requests.
+
+    Deny by default: if INTERNAL_KEY is not configured, reject all requests.
+    """
+    if not settings.INTERNAL_KEY:
+        logger.warning(
+            f"Internal API request rejected from {request.remote_addr}: "
+            "INTERNAL_KEY is not configured"
+        )
+        return jsonify({"error": "Unauthorized", "message": "Internal API is not configured"}), 401
+    internal_key = request.headers.get("X-Internal-Key")
+    if not internal_key or internal_key != settings.INTERNAL_KEY:
+        logger.warning(f"Unauthorized internal API access attempt from {request.remote_addr}")
+        return jsonify({"error": "Unauthorized", "message": "Invalid or missing internal key"}), 401
 
 
 @internal.route("/api/download", methods=["get"])

application/api/user/agents/routes.py

@@ -23,6 +23,8 @@ from application.api.user.base import (
     workflow_nodes_collection,
     workflows_collection,
 )
+from application.storage.db.dual_write import dual_write
+from application.storage.db.repositories.users import UsersRepository
 from application.core.json_schema_utils import (
     JsonSchemaValidationError,
     normalize_json_schema_payload,
@@ -73,6 +75,7 @@ AGENT_TYPE_SCHEMAS = {
             "token_limit",
             "limited_request_mode",
             "request_limit",
+            "allow_system_prompt_override",
             "createdAt",
             "updatedAt",
             "lastUsedAt",
@@ -96,6 +99,7 @@ AGENT_TYPE_SCHEMAS = {
             "token_limit",
             "limited_request_mode",
             "request_limit",
+            "allow_system_prompt_override",
             "createdAt",
             "updatedAt",
             "lastUsedAt",
@@ -220,6 +224,12 @@ def build_agent_document(
         base_doc["request_limit"] = int(
             data.get("request_limit", settings.DEFAULT_AGENT_LIMITS["request_limit"])
         )
+    if "allow_system_prompt_override" in allowed_fields:
+        base_doc["allow_system_prompt_override"] = (
+            data.get("allow_system_prompt_override") == "True"
+            if isinstance(data.get("allow_system_prompt_override"), str)
+            else bool(data.get("allow_system_prompt_override", False))
+        )
     return {k: v for k, v in base_doc.items() if k in allowed_fields}
 
 
@@ -292,6 +302,9 @@ class GetAgent(Resource):
                 "default_model_id": agent.get("default_model_id", ""),
                 "folder_id": agent.get("folder_id"),
                 "workflow": agent.get("workflow"),
+                "allow_system_prompt_override": agent.get(
+                    "allow_system_prompt_override", False
+                ),
             }
             return make_response(jsonify(data), 200)
         except Exception as e:
@@ -373,6 +386,9 @@ class GetAgents(Resource):
                     "default_model_id": agent.get("default_model_id", ""),
                     "folder_id": agent.get("folder_id"),
                     "workflow": agent.get("workflow"),
+                    "allow_system_prompt_override": agent.get(
+                        "allow_system_prompt_override", False
+                    ),
                 }
                 for agent in agents
                 if "source" in agent
@@ -450,6 +466,10 @@ class CreateAgent(Resource):
             "folder_id": fields.String(
                 required=False, description="Folder ID to organize the agent"
             ),
+            "allow_system_prompt_override": fields.Boolean(
+                required=False,
+                description="Allow API callers to override the system prompt via the v1 endpoint",
+            ),
         },
     )
 
@@ -491,9 +511,9 @@ class CreateAgent(Resource):
                 data["json_schema"] = normalize_json_schema_payload(
                     data.get("json_schema")
                 )
-            except JsonSchemaValidationError as exc:
+            except JsonSchemaValidationError:
                 return make_response(
-                    jsonify({"success": False, "message": f"JSON schema {exc}"}),
+                    jsonify({"success": False, "message": "Invalid JSON schema"}),
                     400,
                 )
         if data.get("status") not in ["draft", "published"]:
@@ -674,6 +694,10 @@ class UpdateAgent(Resource):
             "folder_id": fields.String(
                 required=False, description="Folder ID to organize the agent"
             ),
+            "allow_system_prompt_override": fields.Boolean(
+                required=False,
+                description="Allow API callers to override the system prompt via the v1 endpoint",
+            ),
         },
     )
 
@@ -765,6 +789,7 @@ class UpdateAgent(Resource):
             "default_model_id",
             "folder_id",
             "workflow",
+            "allow_system_prompt_override",
         ]
 
         for field in allowed_fields:
@@ -872,9 +897,9 @@ class UpdateAgent(Resource):
                         update_fields[field] = normalize_json_schema_payload(
                             json_schema
                         )
-                    except JsonSchemaValidationError as exc:
+                    except JsonSchemaValidationError:
                         return make_response(
-                            jsonify({"success": False, "message": f"JSON schema {exc}"}),
+                            jsonify({"success": False, "message": "Invalid JSON schema"}),
                             400,
                         )
                 else:
@@ -983,6 +1008,13 @@ class UpdateAgent(Resource):
                 if workflow_error:
                     return workflow_error
                 update_fields[field] = workflow_id
+            elif field == "allow_system_prompt_override":
+                raw_value = data.get("allow_system_prompt_override", False)
+                update_fields[field] = (
+                    raw_value == "True"
+                    if isinstance(raw_value, str)
+                    else bool(raw_value)
+                )
             else:
                 value = data[field]
                 if field in ["name", "description", "prompt_id", "agent_type"]:
@@ -1220,6 +1252,9 @@ class PinnedAgents(Resource):
                     {"user_id": user_id},
                     {"$pullAll": {"agent_preferences.pinned": stale_ids}},
                 )
+                dual_write(UsersRepository,
+                    lambda repo, uid=user_id, ids=stale_ids: repo.remove_pinned_bulk(uid, ids)
+                )
             list_pinned_agents = [
                 {
                     "id": str(agent["_id"]),
@@ -1351,12 +1386,18 @@ class PinAgent(Resource):
                     {"user_id": user_id},
                     {"$pull": {"agent_preferences.pinned": agent_id}},
                 )
+                dual_write(UsersRepository,
+                    lambda repo, uid=user_id, aid=agent_id: repo.remove_pinned(uid, aid)
+                )
                 action = "unpinned"
             else:
                 users_collection.update_one(
                     {"user_id": user_id},
                     {"$addToSet": {"agent_preferences.pinned": agent_id}},
                 )
+                dual_write(UsersRepository,
+                    lambda repo, uid=user_id, aid=agent_id: repo.add_pinned(uid, aid)
+                )
                 action = "pinned"
         except Exception as err:
             current_app.logger.error(f"Error pinning/unpinning agent: {err}")
@@ -1402,6 +1443,9 @@ class RemoveSharedAgent(Resource):
                     }
                 },
             )
+            dual_write(UsersRepository,
+                lambda repo, uid=user_id, aid=agent_id: repo.remove_agent_from_all(uid, aid)
+            )
 
             return make_response(jsonify({"success": True, "action": "removed"}), 200)
         except Exception as err:

application/api/user/agents/sharing.py

@@ -18,6 +18,8 @@ from application.api.user.base import (
     user_tools_collection,
     users_collection,
 )
+from application.storage.db.dual_write import dual_write
+from application.storage.db.repositories.users import UsersRepository
 from application.utils import generate_image_url
 
 agents_sharing_ns = Namespace(
@@ -105,6 +107,9 @@ class SharedAgent(Resource):
                         {"user_id": user_id},
                         {"$addToSet": {"agent_preferences.shared_with_me": agent_id}},
                     )
+                    dual_write(UsersRepository,
+                        lambda repo, uid=user_id, aid=agent_id: repo.add_shared(uid, aid)
+                    )
             return make_response(jsonify(data), 200)
         except Exception as err:
             current_app.logger.error(f"Error retrieving shared agent: {err}")
@@ -139,6 +144,9 @@ class SharedAgents(Resource):
                     {"user_id": user_id},
                     {"$pullAll": {"agent_preferences.shared_with_me": stale_ids}},
                 )
+                dual_write(UsersRepository,
+                    lambda repo, uid=user_id, ids=stale_ids: repo.remove_shared_bulk(uid, ids)
+                )
             pinned_ids = set(user_doc.get("agent_preferences", {}).get("pinned", []))
 
             list_shared_agents = [

application/api/user/attachments/routes.py

@@ -612,6 +612,10 @@ class LiveSpeechToTextFinish(Resource):
 class ServeImage(Resource):
     @api.doc(description="Serve an image from storage")
     def get(self, image_path):
+        if ".." in image_path or image_path.startswith("/") or "\x00" in image_path:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid image path"}), 400
+            )
         try:
             from application.api.user.base import storage
 
@@ -629,6 +633,10 @@ class ServeImage(Resource):
             return make_response(
                 jsonify({"success": False, "message": "Image not found"}), 404
             )
+        except ValueError:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid image path"}), 400
+            )
         except Exception as e:
             current_app.logger.error(f"Error serving image: {e}")
             return make_response(

application/api/user/base.py

@@ -15,6 +15,8 @@ from werkzeug.utils import secure_filename
 
 from application.core.mongo_db import MongoDB
 from application.core.settings import settings
+from application.storage.db.dual_write import dual_write
+from application.storage.db.repositories.users import UsersRepository
 from application.storage.storage_creator import StorageCreator
 from application.vectorstore.vector_creator import VectorCreator
 
@@ -132,6 +134,9 @@ def ensure_user_doc(user_id):
     if updates:
         users_collection.update_one({"user_id": user_id}, {"$set": updates})
         user_doc = users_collection.find_one({"user_id": user_id})
+
+    dual_write(UsersRepository, lambda repo: repo.upsert(user_id))
+
     return user_doc
 
 

application/api/user/sharing/routes.py

@@ -57,7 +57,7 @@ class ShareConversation(Resource):
 
         try:
             conversation = conversations_collection.find_one(
-                {"_id": ObjectId(conversation_id)}
+                {"_id": ObjectId(conversation_id), "user": user}
             )
             if conversation is None:
                 return make_response(

application/api/user/sources/upload.py

@@ -463,6 +463,16 @@ class ManageSourceFiles(Resource):
                 removed_files = []
                 map_updated = False
                 for file_path in file_paths:
+                    if ".." in str(file_path) or str(file_path).startswith("/"):
+                        return make_response(
+                            jsonify(
+                                {
+                                    "success": False,
+                                    "message": "Invalid file path",
+                                }
+                            ),
+                            400,
+                        )
                     full_path = f"{source_file_path}/{file_path}"
 
                     # Remove from storage

application/api/user/tools/mcp.py

@@ -14,6 +14,7 @@ from application.api.user.tools.routes import transform_actions
 from application.cache import get_redis_instance
 from application.core.mongo_db import MongoDB
 from application.core.settings import settings
+from application.core.url_validation import SSRFError, validate_url
 from application.security.encryption import decrypt_credentials, encrypt_credentials
 from application.utils import check_required_fields
 
@@ -63,6 +64,21 @@ def _extract_auth_credentials(config):
     return auth_credentials
 
 
+def _validate_mcp_server_url(config: dict) -> None:
+    """Validate the server_url in an MCP config to prevent SSRF.
+
+    Raises:
+        ValueError: If the URL is missing or points to a blocked address.
+    """
+    server_url = (config.get("server_url") or "").strip()
+    if not server_url:
+        raise ValueError("server_url is required")
+    try:
+        validate_url(server_url)
+    except SSRFError as exc:
+        raise ValueError(f"Invalid server URL: {exc}") from exc
+
+
 @tools_mcp_ns.route("/mcp_server/test")
 class TestMCPServerConfig(Resource):
     @api.expect(
@@ -97,6 +113,8 @@ class TestMCPServerConfig(Resource):
                     400,
                 )
 
+            _validate_mcp_server_url(config)
+
             auth_credentials = _extract_auth_credentials(config)
             test_config = config.copy()
             test_config["auth_credentials"] = auth_credentials
@@ -105,15 +123,41 @@ class TestMCPServerConfig(Resource):
             result = mcp_tool.test_connection()
 
             if result.get("requires_oauth"):
-                return make_response(jsonify(result), 200)
+                safe_result = {
+                    k: v
+                    for k, v in result.items()
+                    if k in ("success", "requires_oauth", "auth_url")
+                }
+                return make_response(jsonify(safe_result), 200)
 
-            if not result.get("success") and "message" in result:
+            if not result.get("success"):
                 current_app.logger.error(
                     f"MCP connection test failed: {result.get('message')}"
                 )
-                result["message"] = "Connection test failed"
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Connection test failed",
+                            "tools_count": 0,
+                        }
+                    ),
+                    200,
+                )
 
-            return make_response(jsonify(result), 200)
+            safe_result = {
+                "success": True,
+                "message": result.get("message", "Connection successful"),
+                "tools_count": result.get("tools_count", 0),
+                "tools": result.get("tools", []),
+            }
+            return make_response(jsonify(safe_result), 200)
+        except ValueError as e:
+            current_app.logger.warning(f"Invalid MCP server test request: {e}")
+            return make_response(
+                jsonify({"success": False, "error": "Invalid MCP server configuration"}),
+                400,
+            )
         except Exception as e:
             current_app.logger.error(f"Error testing MCP server: {e}", exc_info=True)
             return make_response(
@@ -165,6 +209,8 @@ class MCPServerSave(Resource):
                     400,
                 )
 
+            _validate_mcp_server_url(config)
+
             auth_credentials = _extract_auth_credentials(config)
             auth_type = config.get("auth_type", "none")
             mcp_config = config.copy()
@@ -279,6 +325,12 @@ class MCPServerSave(Resource):
                     "tools_count": len(transformed_actions),
                 }
             return make_response(jsonify(response_data), 200)
+        except ValueError as e:
+            current_app.logger.warning(f"Invalid MCP server save request: {e}")
+            return make_response(
+                jsonify({"success": False, "error": "Invalid MCP server configuration"}),
+                400,
+            )
         except Exception as e:
             current_app.logger.error(f"Error saving MCP server: {e}", exc_info=True)
             return make_response(

application/api/user/tools/routes.py

@@ -8,6 +8,7 @@ from application.agents.tools.spec_parser import parse_spec
 from application.agents.tools.tool_manager import ToolManager
 from application.api import api
 from application.api.user.base import user_tools_collection
+from application.core.url_validation import SSRFError, validate_url
 from application.security.encryption import decrypt_credentials, encrypt_credentials
 from application.utils import check_required_fields, validate_function_name
 
@@ -130,6 +131,8 @@ tools_ns = Namespace("tools", description="Tool management operations", path="/a
 class AvailableTools(Resource):
     @api.doc(description="Get available tools for a user")
     def get(self):
+        if not request.decoded_token:
+            return make_response(jsonify({"success": False}), 401)
         try:
             tools_metadata = []
             for tool_name, tool_instance in tool_manager.tools.items():
@@ -236,6 +239,16 @@ class CreateTool(Resource):
         if missing_fields:
             return missing_fields
         try:
+            if data["name"] == "mcp_tool":
+                server_url = (data.get("config", {}).get("server_url") or "").strip()
+                if server_url:
+                    try:
+                        validate_url(server_url)
+                    except SSRFError:
+                        return make_response(
+                            jsonify({"success": False, "message": "Invalid server URL"}),
+                            400,
+                        )
             tool_instance = tool_manager.tools.get(data["name"])
             if not tool_instance:
                 return make_response(
@@ -421,6 +434,16 @@ class UpdateToolConfig(Resource):
                 return make_response(jsonify({"success": False}), 404)
 
             tool_name = tool_doc.get("name")
+            if tool_name == "mcp_tool":
+                server_url = (data["config"].get("server_url") or "").strip()
+                if server_url:
+                    try:
+                        validate_url(server_url)
+                    except SSRFError:
+                        return make_response(
+                            jsonify({"success": False, "message": "Invalid server URL"}),
+                            400,
+                        )
             tool_instance = tool_manager.tools.get(tool_name)
             config_requirements = (
                 tool_instance.get_config_requirements() if tool_instance else {}

application/api/v1/__init__.py

@@ -0,0 +1,3 @@
+from application.api.v1.routes import v1_bp
+
+__all__ = ["v1_bp"]

application/api/v1/routes.py

@@ -0,0 +1,333 @@
+"""Standard chat completions API routes.
+
+Exposes ``/v1/chat/completions`` and ``/v1/models`` endpoints that
+follow the widely-adopted chat completions protocol so external tools
+(opencode, continue, etc.) can connect to DocsGPT agents.
+"""
+
+import json
+import logging
+import time
+import traceback
+from typing import Any, Dict, Generator, Optional
+
+from flask import Blueprint, jsonify, make_response, request, Response
+
+from application.api.answer.routes.base import BaseAnswerResource
+from application.api.answer.services.stream_processor import StreamProcessor
+from application.api.v1.translator import (
+    translate_request,
+    translate_response,
+    translate_stream_event,
+)
+from application.core.mongo_db import MongoDB
+from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+v1_bp = Blueprint("v1", __name__, url_prefix="/v1")
+
+
+def _extract_bearer_token() -> Optional[str]:
+    """Extract API key from Authorization: Bearer header."""
+    auth = request.headers.get("Authorization", "")
+    if auth.startswith("Bearer "):
+        return auth[7:].strip()
+    return None
+
+
+def _lookup_agent(api_key: str) -> Optional[Dict]:
+    """Look up the agent document for this API key."""
+    try:
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        return db["agents"].find_one({"key": api_key})
+    except Exception:
+        logger.warning("Failed to look up agent for API key", exc_info=True)
+        return None
+
+
+def _get_model_name(agent: Optional[Dict], api_key: str) -> str:
+    """Return agent name for display as model name."""
+    if agent:
+        return agent.get("name", api_key)
+    return api_key
+
+
+class _V1AnswerHelper(BaseAnswerResource):
+    """Thin wrapper to access complete_stream / process_response_stream."""
+    pass
+
+
+@v1_bp.route("/chat/completions", methods=["POST"])
+def chat_completions():
+    """Handle POST /v1/chat/completions."""
+    api_key = _extract_bearer_token()
+    if not api_key:
+        return make_response(
+            jsonify({"error": {"message": "Missing Authorization header", "type": "auth_error"}}),
+            401,
+        )
+
+    data = request.get_json()
+    if not data or not data.get("messages"):
+        return make_response(
+            jsonify({"error": {"message": "messages field is required", "type": "invalid_request"}}),
+            400,
+        )
+
+    is_stream = data.get("stream", False)
+    agent_doc = _lookup_agent(api_key)
+    model_name = _get_model_name(agent_doc, api_key)
+
+    try:
+        internal_data = translate_request(data, api_key)
+    except Exception as e:
+        logger.error(f"/v1/chat/completions translate error: {e}", exc_info=True)
+        return make_response(
+            jsonify({"error": {"message": "Failed to process request", "type": "invalid_request"}}),
+            400,
+        )
+
+    # Link decoded_token to the agent's owner so continuation state,
+    # logs, and tool execution use the correct user identity.
+    agent_user = agent_doc.get("user") if agent_doc else None
+    decoded_token = {"sub": agent_user or "api_key_user"}
+
+    try:
+        processor = StreamProcessor(internal_data, decoded_token)
+
+        if internal_data.get("tool_actions"):
+            # Continuation mode
+            conversation_id = internal_data.get("conversation_id")
+            if not conversation_id:
+                return make_response(
+                    jsonify({"error": {"message": "conversation_id required for tool continuation", "type": "invalid_request"}}),
+                    400,
+                )
+            (
+                agent,
+                messages,
+                tools_dict,
+                pending_tool_calls,
+                tool_actions,
+            ) = processor.resume_from_tool_actions(
+                internal_data["tool_actions"], conversation_id
+            )
+            continuation = {
+                "messages": messages,
+                "tools_dict": tools_dict,
+                "pending_tool_calls": pending_tool_calls,
+                "tool_actions": tool_actions,
+            }
+            question = ""
+        else:
+            # Normal mode
+            question = internal_data.get("question", "")
+            agent = processor.build_agent(question)
+            continuation = None
+
+        if not processor.decoded_token:
+            return make_response(
+                jsonify({"error": {"message": "Unauthorized", "type": "auth_error"}}),
+                401,
+            )
+
+        helper = _V1AnswerHelper()
+        usage_error = helper.check_usage(processor.agent_config)
+        if usage_error:
+            return usage_error
+
+        should_save_conversation = bool(internal_data.get("save_conversation", False))
+
+        if is_stream:
+            return Response(
+                _stream_response(
+                    helper,
+                    question,
+                    agent,
+                    processor,
+                    model_name,
+                    continuation,
+                    should_save_conversation,
+                ),
+                mimetype="text/event-stream",
+                headers={
+                    "Cache-Control": "no-cache",
+                    "X-Accel-Buffering": "no",
+                },
+            )
+        else:
+            return _non_stream_response(
+                helper,
+                question,
+                agent,
+                processor,
+                model_name,
+                continuation,
+                should_save_conversation,
+            )
+
+    except ValueError as e:
+        logger.error(
+            f"/v1/chat/completions error: {e} - {traceback.format_exc()}",
+            extra={"error": str(e)},
+        )
+        return make_response(
+            jsonify({"error": {"message": "Failed to process request", "type": "invalid_request"}}),
+            400,
+        )
+    except Exception as e:
+        logger.error(
+            f"/v1/chat/completions error: {e} - {traceback.format_exc()}",
+            extra={"error": str(e)},
+        )
+        return make_response(
+            jsonify({"error": {"message": "Internal server error", "type": "server_error"}}),
+            500,
+        )
+
+
+def _stream_response(
+    helper: _V1AnswerHelper,
+    question: str,
+    agent: Any,
+    processor: StreamProcessor,
+    model_name: str,
+    continuation: Optional[Dict],
+    should_save_conversation: bool,
+) -> Generator[str, None, None]:
+    """Generate translated SSE chunks for streaming response."""
+    completion_id = f"chatcmpl-{int(time.time())}"
+
+    internal_stream = helper.complete_stream(
+        question=question,
+        agent=agent,
+        conversation_id=processor.conversation_id,
+        user_api_key=processor.agent_config.get("user_api_key"),
+        decoded_token=processor.decoded_token,
+        agent_id=processor.agent_id,
+        model_id=processor.model_id,
+        should_save_conversation=should_save_conversation,
+        _continuation=continuation,
+    )
+
+    for line in internal_stream:
+        if not line.strip():
+            continue
+        # Parse the internal SSE event
+        event_str = line.replace("data: ", "").strip()
+        try:
+            event_data = json.loads(event_str)
+        except (json.JSONDecodeError, TypeError):
+            continue
+
+        # Update completion_id when we get the conversation id
+        if event_data.get("type") == "id":
+            conv_id = event_data.get("id", "")
+            if conv_id:
+                completion_id = f"chatcmpl-{conv_id}"
+
+        # Translate to standard format
+        translated = translate_stream_event(event_data, completion_id, model_name)
+        for chunk in translated:
+            yield chunk
+
+
+def _non_stream_response(
+    helper: _V1AnswerHelper,
+    question: str,
+    agent: Any,
+    processor: StreamProcessor,
+    model_name: str,
+    continuation: Optional[Dict],
+    should_save_conversation: bool,
+) -> Response:
+    """Collect full response and return as single JSON."""
+    stream = helper.complete_stream(
+        question=question,
+        agent=agent,
+        conversation_id=processor.conversation_id,
+        user_api_key=processor.agent_config.get("user_api_key"),
+        decoded_token=processor.decoded_token,
+        agent_id=processor.agent_id,
+        model_id=processor.model_id,
+        should_save_conversation=should_save_conversation,
+        _continuation=continuation,
+    )
+
+    result = helper.process_response_stream(stream)
+
+    if result["error"]:
+        return make_response(
+            jsonify({"error": {"message": result["error"], "type": "server_error"}}),
+            500,
+        )
+
+    extra = result.get("extra")
+    pending = extra.get("pending_tool_calls") if isinstance(extra, dict) else None
+
+    response = translate_response(
+        conversation_id=result["conversation_id"],
+        answer=result["answer"] or "",
+        sources=result["sources"],
+        tool_calls=result["tool_calls"],
+        thought=result["thought"] or "",
+        model_name=model_name,
+        pending_tool_calls=pending,
+    )
+    return make_response(jsonify(response), 200)
+
+
+@v1_bp.route("/models", methods=["GET"])
+def list_models():
+    """Handle GET /v1/models — return agents as models."""
+    api_key = _extract_bearer_token()
+    if not api_key:
+        return make_response(
+            jsonify({"error": {"message": "Missing Authorization header", "type": "auth_error"}}),
+            401,
+        )
+
+    try:
+        mongo = MongoDB.get_client()
+        db = mongo[settings.MONGO_DB_NAME]
+        agents_collection = db["agents"]
+
+        # Find the agent for this api_key
+        agent = agents_collection.find_one({"key": api_key})
+        if not agent:
+            return make_response(
+                jsonify({"error": {"message": "Invalid API key", "type": "auth_error"}}),
+                401,
+            )
+
+        user = agent.get("user")
+
+        # Return all agents belonging to this user
+        user_agents = list(agents_collection.find({"user": user}))
+
+        models = []
+        for ag in user_agents:
+            created = ag.get("createdAt")
+            created_ts = int(created.timestamp()) if created else int(time.time())
+            model_id = str(ag.get("_id") or ag.get("id") or "")
+            models.append({
+                "id": model_id,
+                "object": "model",
+                "created": created_ts,
+                "owned_by": "docsgpt",
+                "name": ag.get("name", ""),
+                "description": ag.get("description", ""),
+            })
+
+        return make_response(
+            jsonify({"object": "list", "data": models}),
+            200,
+        )
+    except Exception as e:
+        logger.error(f"/v1/models error: {e}", exc_info=True)
+        return make_response(
+            jsonify({"error": {"message": "Internal server error", "type": "server_error"}}),
+            500,
+        )

application/api/v1/translator.py

@@ -0,0 +1,433 @@
+"""Translate between standard chat completions format and DocsGPT internals.
+
+This module handles:
+- Request translation (chat completions -> DocsGPT internal format)
+- Response translation (DocsGPT response -> chat completions format)
+- Streaming event translation (DocsGPT SSE -> standard SSE chunks)
+"""
+
+import json
+import time
+from typing import Any, Dict, List, Optional
+
+def _get_client_tool_name(tc: Dict) -> str:
+    """Return the original tool name for client-facing responses.
+
+    For client-side tools the ``tool_name`` field carries the name the
+    client originally registered.  Fall back to ``action_name`` (which
+    is now the clean LLM-visible name) or ``name``.
+    """
+    return tc.get("tool_name", tc.get("action_name", tc.get("name", "")))
+
+
+# ---------------------------------------------------------------------------
+# Request translation
+# ---------------------------------------------------------------------------
+
+
+def is_continuation(messages: List[Dict]) -> bool:
+    """Check if messages represent a tool-call continuation.
+
+    A continuation is detected when the last message(s) have ``role: "tool"``
+    immediately after an assistant message with ``tool_calls``.
+    """
+    if not messages:
+        return False
+    # Walk backwards: if we see tool messages before hitting a non-tool, non-assistant message
+    # and there's an assistant message with tool_calls, it's a continuation.
+    i = len(messages) - 1
+    while i >= 0 and messages[i].get("role") == "tool":
+        i -= 1
+    if i < 0:
+        return False
+    return (
+        messages[i].get("role") == "assistant"
+        and bool(messages[i].get("tool_calls"))
+    )
+
+
+def extract_tool_results(messages: List[Dict]) -> List[Dict]:
+    """Extract tool results from trailing tool messages for continuation.
+
+    Returns a list of ``tool_actions`` dicts with ``call_id`` and ``result``.
+    """
+    results = []
+    for msg in reversed(messages):
+        if msg.get("role") != "tool":
+            break
+        call_id = msg.get("tool_call_id", "")
+        content = msg.get("content", "")
+        if isinstance(content, str):
+            try:
+                content = json.loads(content)
+            except (json.JSONDecodeError, TypeError):
+                pass
+        results.append({"call_id": call_id, "result": content})
+    results.reverse()
+    return results
+
+
+def extract_conversation_id(messages: List[Dict]) -> Optional[str]:
+    """Try to extract conversation_id from the assistant message before tool results.
+
+    The conversation_id may be stored in a custom field on the assistant message
+    from a previous response cycle.
+    """
+    for msg in reversed(messages):
+        if msg.get("role") == "assistant":
+            # Check docsgpt extension
+            return msg.get("docsgpt", {}).get("conversation_id")
+    return None
+
+
+def extract_system_prompt(messages: List[Dict]) -> Optional[str]:
+    """Extract the first system message content from the messages array.
+
+    Returns None if no system message is present.
+    """
+    for msg in messages:
+        if msg.get("role") == "system":
+            return msg.get("content", "")
+    return None
+
+
+def convert_history(messages: List[Dict]) -> List[Dict]:
+    """Convert chat completions messages array to DocsGPT history format.
+
+    DocsGPT history is a list of ``{prompt, response}`` dicts.
+    Excludes the last user message (that becomes the ``question``).
+    """
+    history = []
+    i = 0
+    while i < len(messages):
+        msg = messages[i]
+        if msg.get("role") == "system":
+            i += 1
+            continue
+        if msg.get("role") == "user":
+            # Look ahead for assistant response
+            if i + 1 < len(messages) and messages[i + 1].get("role") == "assistant":
+                content = messages[i + 1].get("content") or ""
+                history.append({
+                    "prompt": msg.get("content", ""),
+                    "response": content,
+                })
+                i += 2
+                continue
+            # Last user message without response — skip (it's the question)
+            i += 1
+            continue
+        i += 1
+    return history
+
+
+def translate_request(
+    data: Dict[str, Any], api_key: str
+) -> Dict[str, Any]:
+    """Translate a chat completions request to DocsGPT internal format.
+
+    Args:
+        data: The incoming request body.
+        api_key: Agent API key from the Authorization header.
+
+    Returns:
+        Dict suitable for passing to ``StreamProcessor``.
+    """
+    messages = data.get("messages", [])
+
+    # Check for continuation (tool results after assistant tool_calls)
+    if is_continuation(messages):
+        tool_actions = extract_tool_results(messages)
+        conversation_id = extract_conversation_id(messages)
+        if not conversation_id:
+            conversation_id = data.get("conversation_id")
+        result = {
+            "conversation_id": conversation_id,
+            "tool_actions": tool_actions,
+            "api_key": api_key,
+        }
+        # Carry tools forward for next iteration
+        if data.get("tools"):
+            result["client_tools"] = data["tools"]
+        return result
+
+    # Normal request — extract question from last user message
+    question = ""
+    for msg in reversed(messages):
+        if msg.get("role") == "user":
+            question = msg.get("content", "")
+            break
+
+    history = convert_history(messages)
+    system_prompt_override = extract_system_prompt(messages)
+
+    docsgpt = data.get("docsgpt", {})
+
+    result = {
+        "question": question,
+        "api_key": api_key,
+        "history": json.dumps(history),
+        # Conversations are NOT persisted by default on the v1 endpoint.
+        # Callers opt in via ``docsgpt.save_conversation: true``.
+        "save_conversation": bool(docsgpt.get("save_conversation", False)),
+    }
+
+    if system_prompt_override is not None:
+        result["system_prompt_override"] = system_prompt_override
+
+    # Client tools
+    if data.get("tools"):
+        result["client_tools"] = data["tools"]
+
+    # DocsGPT extensions
+    if docsgpt.get("attachments"):
+        result["attachments"] = docsgpt["attachments"]
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Response translation (non-streaming)
+# ---------------------------------------------------------------------------
+
+
+def translate_response(
+    conversation_id: str,
+    answer: str,
+    sources: Optional[List[Dict]],
+    tool_calls: Optional[List[Dict]],
+    thought: str,
+    model_name: str,
+    pending_tool_calls: Optional[List[Dict]] = None,
+) -> Dict[str, Any]:
+    """Translate DocsGPT response to chat completions format.
+
+    Args:
+        conversation_id: The DocsGPT conversation ID.
+        answer: The assistant's text response.
+        sources: RAG retrieval sources.
+        tool_calls: Completed tool call results.
+        thought: Reasoning/thinking tokens.
+        model_name: Model/agent identifier.
+        pending_tool_calls: Pending client-side tool calls (if paused).
+
+    Returns:
+        Dict in the standard chat completions response format.
+    """
+    created = int(time.time())
+    completion_id = f"chatcmpl-{conversation_id}" if conversation_id else f"chatcmpl-{created}"
+
+    # Build message
+    message: Dict[str, Any] = {"role": "assistant"}
+
+    if pending_tool_calls:
+        # Tool calls pending — return them for client execution
+        message["content"] = None
+        message["tool_calls"] = [
+            {
+                "id": tc.get("call_id", ""),
+                "type": "function",
+                "function": {
+                    "name": _get_client_tool_name(tc),
+                    "arguments": (
+                        json.dumps(tc["arguments"])
+                        if isinstance(tc.get("arguments"), dict)
+                        else tc.get("arguments", "{}")
+                    ),
+                },
+            }
+            for tc in pending_tool_calls
+        ]
+        finish_reason = "tool_calls"
+    else:
+        message["content"] = answer
+        if thought:
+            message["reasoning_content"] = thought
+        finish_reason = "stop"
+
+    result: Dict[str, Any] = {
+        "id": completion_id,
+        "object": "chat.completion",
+        "created": created,
+        "model": model_name,
+        "choices": [
+            {
+                "index": 0,
+                "message": message,
+                "finish_reason": finish_reason,
+            }
+        ],
+        "usage": {
+            "prompt_tokens": 0,
+            "completion_tokens": 0,
+            "total_tokens": 0,
+        },
+    }
+
+    # DocsGPT extensions
+    docsgpt: Dict[str, Any] = {}
+    if conversation_id:
+        docsgpt["conversation_id"] = conversation_id
+    if sources:
+        docsgpt["sources"] = sources
+    if tool_calls:
+        docsgpt["tool_calls"] = tool_calls
+    if docsgpt:
+        result["docsgpt"] = docsgpt
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Streaming event translation
+# ---------------------------------------------------------------------------
+
+
+def _make_chunk(
+    completion_id: str,
+    model_name: str,
+    delta: Dict[str, Any],
+    finish_reason: Optional[str] = None,
+) -> str:
+    """Build a single SSE chunk in the standard streaming format."""
+    chunk = {
+        "id": completion_id,
+        "object": "chat.completion.chunk",
+        "created": int(time.time()),
+        "model": model_name,
+        "choices": [
+            {
+                "index": 0,
+                "delta": delta,
+                "finish_reason": finish_reason,
+            }
+        ],
+    }
+    return f"data: {json.dumps(chunk)}\n\n"
+
+
+def _make_docsgpt_chunk(data: Dict[str, Any]) -> str:
+    """Build a DocsGPT extension SSE chunk."""
+    return f"data: {json.dumps({'docsgpt': data})}\n\n"
+
+
+def translate_stream_event(
+    event_data: Dict[str, Any],
+    completion_id: str,
+    model_name: str,
+) -> List[str]:
+    """Translate a DocsGPT SSE event dict to standard streaming chunks.
+
+    May return 0, 1, or 2 chunks per input event. For example, a completed
+    tool call produces both a docsgpt extension chunk and nothing on the
+    standard side (since server-side tool calls aren't surfaced in standard
+    format).
+
+    Args:
+        event_data: Parsed DocsGPT event dict.
+        completion_id: The completion ID for this response.
+        model_name: Model/agent identifier.
+
+    Returns:
+        List of SSE-formatted strings to send to the client.
+    """
+    event_type = event_data.get("type")
+    chunks: List[str] = []
+
+    if event_type == "answer":
+        chunks.append(
+            _make_chunk(completion_id, model_name, {"content": event_data.get("answer", "")})
+        )
+
+    elif event_type == "thought":
+        chunks.append(
+            _make_chunk(
+                completion_id, model_name,
+                {"reasoning_content": event_data.get("thought", "")},
+            )
+        )
+
+    elif event_type == "source":
+        chunks.append(
+            _make_docsgpt_chunk({
+                "type": "source",
+                "sources": event_data.get("source", []),
+            })
+        )
+
+    elif event_type == "tool_call":
+        tc_data = event_data.get("data", {})
+        status = tc_data.get("status")
+
+        if status == "requires_client_execution":
+            # Standard: stream as tool_calls delta
+            args = tc_data.get("arguments", {})
+            args_str = json.dumps(args) if isinstance(args, dict) else str(args)
+            chunks.append(
+                _make_chunk(completion_id, model_name, {
+                    "tool_calls": [{
+                        "index": 0,
+                        "id": tc_data.get("call_id", ""),
+                        "type": "function",
+                        "function": {
+                            "name": _get_client_tool_name(tc_data),
+                            "arguments": args_str,
+                        },
+                    }],
+                })
+            )
+        elif status == "awaiting_approval":
+            # Extension: approval needed
+            chunks.append(_make_docsgpt_chunk({"type": "tool_call", "data": tc_data}))
+        elif status in ("completed", "pending", "error", "denied", "skipped"):
+            # Extension: tool call progress
+            chunks.append(_make_docsgpt_chunk({"type": "tool_call", "data": tc_data}))
+
+    elif event_type == "tool_calls_pending":
+        # Standard: finish_reason = tool_calls
+        chunks.append(
+            _make_chunk(completion_id, model_name, {}, finish_reason="tool_calls")
+        )
+        # Also emit as docsgpt extension
+        chunks.append(
+            _make_docsgpt_chunk({
+                "type": "tool_calls_pending",
+                "pending_tool_calls": event_data.get("data", {}).get("pending_tool_calls", []),
+            })
+        )
+
+    elif event_type == "end":
+        chunks.append(
+            _make_chunk(completion_id, model_name, {}, finish_reason="stop")
+        )
+        chunks.append("data: [DONE]\n\n")
+
+    elif event_type == "id":
+        chunks.append(
+            _make_docsgpt_chunk({
+                "type": "id",
+                "conversation_id": event_data.get("id", ""),
+            })
+        )
+
+    elif event_type == "error":
+        # Emit as standard error (non-standard but widely supported)
+        error_data = {
+            "error": {
+                "message": event_data.get("error", "An error occurred"),
+                "type": "server_error",
+            }
+        }
+        chunks.append(f"data: {json.dumps(error_data)}\n\n")
+
+    elif event_type == "structured_answer":
+        chunks.append(
+            _make_chunk(
+                completion_id, model_name,
+                {"content": event_data.get("answer", "")},
+            )
+        )
+
+    # Skip: tool_calls (redundant), research_plan, research_progress
+
+    return chunks

application/app.py

@@ -17,6 +17,7 @@ from application.api.answer import answer  # noqa: E402
 from application.api.internal.routes import internal  # noqa: E402
 from application.api.user.routes import user  # noqa: E402
 from application.api.connector.routes import connector  # noqa: E402
+from application.api.v1 import v1_bp  # noqa: E402
 from application.celery_init import celery  # noqa: E402
 from application.core.settings import settings  # noqa: E402
 from application.stt.upload_limits import (  # noqa: E402
@@ -36,6 +37,7 @@ app.register_blueprint(user)
 app.register_blueprint(answer)
 app.register_blueprint(internal)
 app.register_blueprint(connector)
+app.register_blueprint(v1_bp)
 app.config.update(
     UPLOAD_FOLDER="inputs",
     CELERY_BROKER_URL=settings.CELERY_BROKER_URL,

application/celery_init.py

@@ -1,6 +1,6 @@
 from celery import Celery
 from application.core.settings import settings
-from celery.signals import setup_logging
+from celery.signals import setup_logging, worker_process_init
 
 
 def make_celery(app_name=__name__):
@@ -20,5 +20,24 @@ def config_loggers(*args, **kwargs):
     setup_logging()
 
 
+@worker_process_init.connect
+def _dispose_db_engine_on_fork(*args, **kwargs):
+    """Dispose the SQLAlchemy engine pool in each forked Celery worker.
+
+    SQLAlchemy connection pools are not fork-safe: file descriptors shared
+    between the parent and a forked worker will corrupt the pool. Disposing
+    on ``worker_process_init`` gives every worker its own fresh pool on
+    first use.
+
+    Imported lazily so Celery workers that don't touch Postgres (or where
+    ``POSTGRES_URI`` is unset) don't fail at startup.
+    """
+    try:
+        from application.storage.db.engine import dispose_engine
+    except Exception:
+        return
+    dispose_engine()
+
+
 celery = make_celery()
 celery.config_from_object("application.celeryconfig")

application/core/db_uri.py

@@ -0,0 +1,89 @@
+"""Normalize user-supplied Postgres URIs for different drivers.
+
+DocsGPT has two Postgres connection strings pointing at potentially
+different databases:
+
+* ``POSTGRES_URI`` feeds SQLAlchemy, which needs the
+  ``postgresql+psycopg://`` dialect prefix to pick the psycopg v3 driver.
+* ``PGVECTOR_CONNECTION_STRING`` feeds ``psycopg.connect()`` directly
+  (via libpq) in ``application/vectorstore/pgvector.py``. libpq only
+  understands ``postgres://`` and ``postgresql://`` — the SQLAlchemy
+  dialect prefix is an invalid URI from its point of view.
+
+The two fields therefore need opposite normalization so operators don't
+have to know which driver a given field feeds. Each normalizer also
+silently upgrades the legacy ``postgresql+psycopg2://`` prefix since
+psycopg2 is no longer in the project.
+
+This module is deliberately separate from ``application/core/settings.py``
+so the Settings class stays focused on field declarations, and the
+URI-rewriting logic can be unit-tested without triggering ``.env``
+file loading from importing Settings.
+"""
+
+from __future__ import annotations
+
+
+def _rewrite_uri_prefixes(v, rewrites):
+    """Shared URI prefix rewriter used by both normalizers below.
+
+    Strips whitespace, returns ``None`` for empty / ``"none"`` values,
+    applies the first matching rewrite, and passes unrecognised input
+    through so downstream consumers (SQLAlchemy, libpq) can produce
+    their own error messages rather than us silently eating a
+    misconfiguration.
+    """
+    if v is None:
+        return None
+    if not isinstance(v, str):
+        return v
+    v = v.strip()
+    if not v or v.lower() == "none":
+        return None
+    for prefix, target in rewrites:
+        if v.startswith(prefix):
+            return target + v[len(prefix):]
+    return v
+
+
+# POSTGRES_URI feeds SQLAlchemy, which needs a ``postgresql+psycopg://``
+# dialect prefix to select the psycopg v3 driver. Normalize the
+# operator-friendly forms TOWARD that dialect.
+_POSTGRES_URI_REWRITES = (
+    ("postgresql+psycopg2://", "postgresql+psycopg://"),
+    ("postgresql://", "postgresql+psycopg://"),
+    ("postgres://", "postgresql+psycopg://"),
+)
+
+
+# PGVECTOR_CONNECTION_STRING feeds ``psycopg.connect()`` directly in
+# application/vectorstore/pgvector.py — NOT SQLAlchemy. libpq only
+# understands ``postgres://`` and ``postgresql://``; the SQLAlchemy
+# dialect prefix is an invalid URI from libpq's point of view. Strip it
+# if the operator accidentally copied their POSTGRES_URI value here.
+_PGVECTOR_CONNECTION_STRING_REWRITES = (
+    ("postgresql+psycopg2://", "postgresql://"),
+    ("postgresql+psycopg://", "postgresql://"),
+)
+
+
+def normalize_postgres_uri(v):
+    """Normalize a user-supplied POSTGRES_URI to the SQLAlchemy psycopg3 form.
+
+    Accepts the forms operators naturally write (``postgres://``,
+    ``postgresql://``) and rewrites them to ``postgresql+psycopg://``.
+    Unknown schemes pass through unchanged so SQLAlchemy can produce its
+    own dialect-not-found error.
+    """
+    return _rewrite_uri_prefixes(v, _POSTGRES_URI_REWRITES)
+
+
+def normalize_pgvector_connection_string(v):
+    """Normalize a user-supplied PGVECTOR_CONNECTION_STRING for libpq.
+
+    Strips the SQLAlchemy dialect prefix if the operator accidentally
+    copied their POSTGRES_URI value here — libpq can't parse it.
+    User-friendly forms (``postgres://``, ``postgresql://``) pass
+    through unchanged since libpq accepts them natively.
+    """
+    return _rewrite_uri_prefixes(v, _PGVECTOR_CONNECTION_STRING_REWRITES)

application/core/model_configs.py

@@ -27,6 +27,8 @@ ANTHROPIC_ATTACHMENTS = IMAGE_ATTACHMENTS
 
 OPENROUTER_ATTACHMENTS = IMAGE_ATTACHMENTS
 
+NOVITA_ATTACHMENTS = IMAGE_ATTACHMENTS
+
 
 OPENAI_MODELS = [
     AvailableModel(
@@ -193,6 +195,46 @@ OPENROUTER_MODELS = [
     ),
 ]
 
+NOVITA_MODELS = [
+    AvailableModel(
+        id="moonshotai/kimi-k2.5",
+        provider=ModelProvider.NOVITA,
+        display_name="Kimi K2.5",
+        description="MoE model with function calling, structured output, reasoning, and vision",
+        capabilities=ModelCapabilities(
+            supports_tools=True,
+            supports_structured_output=True,
+            supported_attachment_types=NOVITA_ATTACHMENTS,
+            context_window=262144,
+        ),
+    ),
+    AvailableModel(
+        id="zai-org/glm-5",
+        provider=ModelProvider.NOVITA,
+        display_name="GLM-5",
+        description="MoE model with function calling, structured output, and reasoning",
+        capabilities=ModelCapabilities(
+            supports_tools=True,
+            supports_structured_output=True,
+            supported_attachment_types=[],
+            context_window=202800,
+        ),
+    ),
+    AvailableModel(
+        id="minimax/minimax-m2.5",
+        provider=ModelProvider.NOVITA,
+        display_name="MiniMax M2.5",
+        description="MoE model with function calling, structured output, and reasoning",
+        capabilities=ModelCapabilities(
+            supports_tools=True,
+            supports_structured_output=True,
+            supported_attachment_types=[],
+            context_window=204800,
+        ),
+    ),
+]
+
+
 AZURE_OPENAI_MODELS = [
     AvailableModel(
         id="azure-gpt-4",

application/core/model_settings.py

@@ -114,6 +114,10 @@ class ModelRegistry:
             settings.LLM_PROVIDER == "openrouter" and settings.API_KEY
         ):
             self._add_openrouter_models(settings)
+        if settings.NOVITA_API_KEY or (
+            settings.LLM_PROVIDER == "novita" and settings.API_KEY
+        ):
+            self._add_novita_models(settings)
         if settings.HUGGINGFACE_API_KEY or (
             settings.LLM_PROVIDER == "huggingface" and settings.API_KEY
         ):
@@ -245,6 +249,21 @@ class ModelRegistry:
         for model in OPENROUTER_MODELS:
             self.models[model.id] = model
 
+    def _add_novita_models(self, settings):
+        from application.core.model_configs import NOVITA_MODELS
+
+        if settings.NOVITA_API_KEY:
+            for model in NOVITA_MODELS:
+                self.models[model.id] = model
+            return
+        if settings.LLM_PROVIDER == "novita" and settings.LLM_NAME:
+            for model in NOVITA_MODELS:
+                if model.id == settings.LLM_NAME:
+                    self.models[model.id] = model
+                    return
+        for model in NOVITA_MODELS:
+            self.models[model.id] = model
+
     def _add_docsgpt_models(self, settings):
         model_id = "docsgpt-local"
         model = AvailableModel(

application/core/model_utils.py

@@ -10,6 +10,7 @@ def get_api_key_for_provider(provider: str) -> Optional[str]:
     provider_key_map = {
         "openai": settings.OPENAI_API_KEY,
         "openrouter": settings.OPEN_ROUTER_API_KEY,
+        "novita": settings.NOVITA_API_KEY,
         "anthropic": settings.ANTHROPIC_API_KEY,
         "google": settings.GOOGLE_API_KEY,
         "groq": settings.GROQ_API_KEY,

application/core/settings.py

@@ -5,8 +5,12 @@ from typing import Optional
 from pydantic import field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-current_dir = os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from application.core.db_uri import (  # noqa: E402
+    normalize_pgvector_connection_string,
+    normalize_postgres_uri,
 )
 
 
@@ -15,19 +19,22 @@ class Settings(BaseSettings):
 
     AUTH_TYPE: Optional[str] = None  # simple_jwt, session_jwt, or None
     LLM_PROVIDER: str = "docsgpt"
-    LLM_NAME: Optional[str] = (
-        None  # if LLM_PROVIDER is openai, LLM_NAME can be gpt-4 or gpt-3.5-turbo
-    )
+    LLM_NAME: Optional[str] = None  # if LLM_PROVIDER is openai, LLM_NAME can be gpt-4 or gpt-3.5-turbo
     EMBEDDINGS_NAME: str = "huggingface_sentence-transformers/all-mpnet-base-v2"
     EMBEDDINGS_BASE_URL: Optional[str] = None  # Remote embeddings API URL (OpenAI-compatible)
-    EMBEDDINGS_KEY: Optional[str] = (
-        None  # api key for embeddings (if using openai, just copy API_KEY)
-    )
-    
+    EMBEDDINGS_KEY: Optional[str] = None  # api key for embeddings (if using openai, just copy API_KEY)
+
     CELERY_BROKER_URL: str = "redis://localhost:6379/0"
     CELERY_RESULT_BACKEND: str = "redis://localhost:6379/1"
     MONGO_URI: str = "mongodb://localhost:27017/docsgpt"
     MONGO_DB_NAME: str = "docsgpt"
+    # User-data Postgres DB Optional during the MongoDB→Postgres migration; becomes required once the migration is
+    # complete.
+    POSTGRES_URI: Optional[str] = None
+
+    # MongoDB→Postgres migration switches
+    USE_POSTGRES: bool = False
+    READ_POSTGRES: bool = False
     LLM_PATH: str = os.path.join(current_dir, "models/docsgpt-7b-f16.gguf")
     DEFAULT_MAX_HISTORY: int = 150
     DEFAULT_LLM_TOKEN_LIMIT: int = 128000  # Fallback when model not found in registry
@@ -45,9 +52,7 @@ class Settings(BaseSettings):
     PARSE_IMAGE_REMOTE: bool = False
     DOCLING_OCR_ENABLED: bool = False  # Enable OCR for docling parsers (PDF, images)
     DOCLING_OCR_ATTACHMENTS_ENABLED: bool = False  # Enable OCR for docling when parsing attachments
-    VECTOR_STORE: str = (
-        "faiss"  #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb" or "pgvector"
-    )
+    VECTOR_STORE: str = "faiss"  #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb" or "pgvector"
     RETRIEVERS_ENABLED: list = ["classic_rag"]
     AGENT_NAME: str = "classic"
     FALLBACK_LLM_PROVIDER: Optional[str] = None  # provider for fallback llm
@@ -55,12 +60,8 @@ class Settings(BaseSettings):
     FALLBACK_LLM_API_KEY: Optional[str] = None  # api key for fallback llm
 
     # Google Drive integration
-    GOOGLE_CLIENT_ID: Optional[str] = (
-        None  # Replace with your actual Google OAuth client ID
-    )
-    GOOGLE_CLIENT_SECRET: Optional[str] = (
-        None  # Replace with your actual Google OAuth client secret
-    )
+    GOOGLE_CLIENT_ID: Optional[str] = None  # Replace with your actual Google OAuth client ID
+    GOOGLE_CLIENT_SECRET: Optional[str] = None  # Replace with your actual Google OAuth client secret
     CONNECTOR_REDIRECT_BASE_URI: Optional[str] = (
         "http://127.0.0.1:7091/api/connectors/callback"  ##add redirect url as it is to your provider's console(gcp)
     )
@@ -72,7 +73,7 @@ class Settings(BaseSettings):
     MICROSOFT_AUTHORITY: Optional[str] = None  # e.g., "https://login.microsoftonline.com/{tenant_id}"
 
     # GitHub source
-    GITHUB_ACCESS_TOKEN: Optional[str] = None # PAT token with read repo access
+    GITHUB_ACCESS_TOKEN: Optional[str] = None  # PAT token with read repo access
 
     # LLM Cache
     CACHE_REDIS_URL: str = "redis://localhost:6379/2"
@@ -90,16 +91,13 @@ class Settings(BaseSettings):
     GROQ_API_KEY: Optional[str] = None
     HUGGINGFACE_API_KEY: Optional[str] = None
     OPEN_ROUTER_API_KEY: Optional[str] = None
+    NOVITA_API_KEY: Optional[str] = None
 
     OPENAI_API_BASE: Optional[str] = None  # azure openai api base url
     OPENAI_API_VERSION: Optional[str] = None  # azure openai api version
     AZURE_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for answering
-    AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = (
-        None  # azure deployment name for embeddings
-    )
-    OPENAI_BASE_URL: Optional[str] = (
-        None  # openai base url for open ai compatable models
-    )
+    AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = None  # azure deployment name for embeddings
+    OPENAI_BASE_URL: Optional[str] = None  # openai base url for open ai compatable models
 
     # elasticsearch
     ELASTIC_CLOUD_ID: Optional[str] = None  # cloud id for elasticsearch
@@ -132,7 +130,10 @@ class Settings(BaseSettings):
     QDRANT_PATH: Optional[str] = None
     QDRANT_DISTANCE_FUNC: str = "Cosine"
 
-    # PGVector vectorstore config
+    # PGVector vectorstore config. Write the URI in whichever form you
+    # prefer — ``postgres://``, ``postgresql://``, or even the SQLAlchemy
+    # dialect form (``postgresql+psycopg://``) are all accepted and
+    # normalized internally for ``psycopg.connect()``.
     PGVECTOR_CONNECTION_STRING: Optional[str] = None
     # Milvus vectorstore config
     MILVUS_COLLECTION_NAME: Optional[str] = "docsgpt"
@@ -141,9 +142,7 @@ class Settings(BaseSettings):
 
     # LanceDB vectorstore config
     LANCEDB_PATH: str = "./data/lancedb"  # Path where LanceDB stores its local data
-    LANCEDB_TABLE_NAME: Optional[str] = (
-        "docsgpts"  # Name of the table to use for storing vectors
-    )
+    LANCEDB_TABLE_NAME: Optional[str] = "docsgpts"  # Name of the table to use for storing vectors
 
     FLASK_DEBUG_MODE: bool = False
     STORAGE_TYPE: str = "local"  # local or s3
@@ -173,6 +172,16 @@ class Settings(BaseSettings):
     COMPRESSION_PROMPT_VERSION: str = "v1.0"  # Track prompt iterations
     COMPRESSION_MAX_HISTORY_POINTS: int = 3  # Keep only last N compression points to prevent DB bloat
 
+    @field_validator("POSTGRES_URI", mode="before")
+    @classmethod
+    def _normalize_postgres_uri_validator(cls, v):
+        return normalize_postgres_uri(v)
+
+    @field_validator("PGVECTOR_CONNECTION_STRING", mode="before")
+    @classmethod
+    def _normalize_pgvector_connection_string_validator(cls, v):
+        return normalize_pgvector_connection_string(v)
+
     @field_validator(
         "API_KEY",
         "OPENAI_API_KEY",
@@ -180,6 +189,7 @@ class Settings(BaseSettings):
         "GOOGLE_API_KEY",
         "GROQ_API_KEY",
         "HUGGINGFACE_API_KEY",
+        "NOVITA_API_KEY",
         "EMBEDDINGS_KEY",
         "FALLBACK_LLM_API_KEY",
         "QDRANT_API_KEY",

application/llm/google_ai.py

@@ -167,6 +167,8 @@ class GoogleLLM(BaseLLM):
                 return "\n".join(parts)
             return ""
 
+        import json as _json
+
         for message in messages:
             role = message.get("role")
             content = message.get("content")
@@ -180,9 +182,66 @@ class GoogleLLM(BaseLLM):
 
             if role == "assistant":
                 role = "model"
-            elif role == "tool":
-                role = "model"
+
             parts = []
+
+            # Standard format: assistant message with tool_calls array
+            msg_tool_calls = message.get("tool_calls")
+            if msg_tool_calls and role == "model":
+                for tc in msg_tool_calls:
+                    func = tc.get("function", {})
+                    args = func.get("arguments", "{}")
+                    if isinstance(args, str):
+                        try:
+                            args = _json.loads(args)
+                        except (_json.JSONDecodeError, TypeError):
+                            args = {}
+                    cleaned_args = self._remove_null_values(args)
+                    thought_sig = tc.get("thought_signature")
+                    if thought_sig:
+                        parts.append(
+                            types.Part(
+                                functionCall=types.FunctionCall(
+                                    name=func.get("name", ""),
+                                    args=cleaned_args,
+                                ),
+                                thoughtSignature=thought_sig,
+                            )
+                        )
+                    else:
+                        parts.append(
+                            types.Part.from_function_call(
+                                name=func.get("name", ""),
+                                args=cleaned_args,
+                            )
+                        )
+                if parts:
+                    cleaned_messages.append(types.Content(role=role, parts=parts))
+                continue
+
+            # Standard format: tool message with tool_call_id
+            tool_call_id = message.get("tool_call_id")
+            if role == "tool" and tool_call_id is not None:
+                result_content = content
+                if isinstance(result_content, str):
+                    try:
+                        result_content = _json.loads(result_content)
+                    except (_json.JSONDecodeError, TypeError):
+                        pass
+                # Google expects function_response name — extract from tool_call_id context
+                # We use a placeholder name since Google API doesn't require exact match
+                parts.append(
+                    types.Part.from_function_response(
+                        name="tool_result",
+                        response={"result": result_content},
+                    )
+                )
+                cleaned_messages.append(types.Content(role="model", parts=parts))
+                continue
+
+            if role == "tool":
+                role = "model"
+
             if role and content is not None:
                 if isinstance(content, str):
                     parts = [types.Part.from_text(text=content)]
@@ -191,15 +250,11 @@ class GoogleLLM(BaseLLM):
                         if "text" in item:
                             parts.append(types.Part.from_text(text=item["text"]))
                         elif "function_call" in item:
-                            # Remove null values from args to avoid API errors
-
+                            # Legacy format support
                             cleaned_args = self._remove_null_values(
                                 item["function_call"]["args"]
                             )
-                            # Create function call part with thought_signature if present
-                            # For Gemini 3 models, we need to include thought_signature
                             if "thought_signature" in item:
-                                # Use Part constructor with functionCall and thoughtSignature
                                 parts.append(
                                     types.Part(
                                         functionCall=types.FunctionCall(
@@ -210,7 +265,6 @@ class GoogleLLM(BaseLLM):
                                     )
                                 )
                             else:
-                                # Use helper method when no thought_signature
                                 parts.append(
                                     types.Part.from_function_call(
                                         name=item["function_call"]["name"],

application/llm/handlers/base.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import uuid
 from abc import ABC, abstractmethod
@@ -315,10 +316,34 @@ class LLMHandler(ABC):
                 current_prompt = self._extract_text_from_content(content)
 
             elif role in {"assistant", "model"}:
-                # If this assistant turn contains tool calls, collect them; otherwise commit a response.
+                # Standard format: tool_calls array on assistant message
+                msg_tool_calls = message.get("tool_calls")
+                if msg_tool_calls:
+                    for tc in msg_tool_calls:
+                        call_id = tc.get("id") or str(uuid.uuid4())
+                        func = tc.get("function", {})
+                        args = func.get("arguments")
+                        if isinstance(args, str):
+                            try:
+                                args = json.loads(args)
+                            except (json.JSONDecodeError, TypeError):
+                                pass
+                        current_tool_calls[call_id] = {
+                            "tool_name": "unknown_tool",
+                            "action_name": func.get("name"),
+                            "arguments": args,
+                            "result": None,
+                            "status": "called",
+                            "call_id": call_id,
+                        }
+                    continue
+
+                # Legacy format: function_call/function_response in content list
                 if isinstance(content, list):
+                    has_fc = False
                     for item in content:
                         if "function_call" in item:
+                            has_fc = True
                             fc = item["function_call"]
                             call_id = fc.get("call_id") or str(uuid.uuid4())
                             current_tool_calls[call_id] = {
@@ -329,37 +354,30 @@ class LLMHandler(ABC):
                                 "status": "called",
                                 "call_id": call_id,
                             }
-                        elif "function_response" in item:
-                            fr = item["function_response"]
-                            call_id = fr.get("call_id") or str(uuid.uuid4())
-                            current_tool_calls[call_id] = {
-                                "tool_name": "unknown_tool",
-                                "action_name": fr.get("name"),
-                                "arguments": None,
-                                "result": fr.get("response", {}).get("result"),
-                                "status": "completed",
-                                "call_id": call_id,
-                            }
-                    # No direct assistant text here; continue to next message
-                    continue
+                    if has_fc:
+                        continue
 
                 response_text = self._extract_text_from_content(content)
                 _commit_query(response_text)
 
             elif role == "tool":
-                # Attach tool outputs to the latest pending tool call if possible
+                # Standard format: tool_call_id on tool message
+                call_id = message.get("tool_call_id")
                 tool_text = self._extract_text_from_content(content)
-                # Attempt to parse function_response style
-                call_id = None
-                if isinstance(content, list):
-                    for item in content:
-                        if "function_response" in item and item["function_response"].get("call_id"):
-                            call_id = item["function_response"]["call_id"]
-                            break
+
                 if call_id and call_id in current_tool_calls:
                     current_tool_calls[call_id]["result"] = tool_text
                     current_tool_calls[call_id]["status"] = "completed"
-                elif queries:
+                # Legacy: function_response in content list
+                elif isinstance(content, list):
+                    for item in content:
+                        if "function_response" in item:
+                            legacy_id = item["function_response"].get("call_id")
+                            if legacy_id and legacy_id in current_tool_calls:
+                                current_tool_calls[legacy_id]["result"] = tool_text
+                                current_tool_calls[legacy_id]["status"] = "completed"
+                                break
+                elif call_id is None and queries:
                     queries[-1].setdefault("tool_calls", []).append(
                         {
                             "tool_name": "unknown_tool",
@@ -648,6 +666,13 @@ class LLMHandler(ABC):
         """
         Execute tool calls and update conversation history.
 
+        When a tool requires approval or client-side execution, it is
+        collected as a pending action instead of being executed.  The
+        generator returns ``(updated_messages, pending_actions)`` where
+        *pending_actions* is ``None`` when every tool was executed
+        normally, or a list of dicts describing actions the client must
+        resolve before the LLM loop can continue.
+
         Args:
             agent: The agent instance
             tool_calls: List of tool calls to execute
@@ -655,9 +680,11 @@ class LLMHandler(ABC):
             messages: Current conversation history
 
         Returns:
-            Updated messages list
+            Tuple of (updated_messages, pending_actions).
+            pending_actions is None if all tools executed, otherwise a list.
         """
         updated_messages = messages.copy()
+        pending_actions: List[Dict] = []
 
         for i, call in enumerate(tool_calls):
             # Check context limit before executing tool call
@@ -763,6 +790,29 @@ class LLMHandler(ABC):
                     # Set flag on agent
                     agent.context_limit_reached = True
                     break
+
+            # ---- Pause check: approval / client-side execution ----
+            llm_class = agent.llm.__class__.__name__
+            pause_info = agent.tool_executor.check_pause(
+                tools_dict, call, llm_class
+            )
+            if pause_info:
+                # Yield pause event so the client knows this tool is waiting
+                yield {
+                    "type": "tool_call",
+                    "data": {
+                        "tool_name": pause_info["tool_name"],
+                        "call_id": pause_info["call_id"],
+                        "action_name": pause_info.get("llm_name", pause_info["name"]),
+                        "arguments": pause_info["arguments"],
+                        "status": pause_info["pause_type"],
+                    },
+                }
+                pending_actions.append(pause_info)
+                # Do NOT add messages for pending tools here.
+                # They will be added on resume to keep call/result pairs together.
+                continue
+
             try:
                 self.tool_calls.append(call)
                 tool_executor_gen = agent._execute_tool_action(tools_dict, call)
@@ -772,25 +822,30 @@ class LLMHandler(ABC):
                     except StopIteration as e:
                         tool_response, call_id = e.value
                         break
-                    
-                function_call_content = {
-                    "function_call": {
-                        "name": call.name,
-                        "args": call.arguments,
-                        "call_id": call_id,
-                    }
-                }
-                # Include thought_signature for Google Gemini 3 models
-                # It should be at the same level as function_call, not inside it
-                if call.thought_signature:
-                    function_call_content["thought_signature"] = call.thought_signature
-                updated_messages.append(
-                    {
-                        "role": "assistant",
-                        "content": [function_call_content],
-                    }
-                )
 
+                # Standard internal format: assistant message with tool_calls array
+                args_str = (
+                    json.dumps(call.arguments)
+                    if isinstance(call.arguments, dict)
+                    else call.arguments
+                )
+                tool_call_obj = {
+                    "id": call_id,
+                    "type": "function",
+                    "function": {
+                        "name": call.name,
+                        "arguments": args_str,
+                    },
+                }
+                # Preserve thought_signature for Google Gemini 3 models
+                if call.thought_signature:
+                    tool_call_obj["thought_signature"] = call.thought_signature
+
+                updated_messages.append({
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [tool_call_obj],
+                })
 
                 updated_messages.append(self.create_tool_message(call, tool_response))
             except Exception as e:
@@ -802,16 +857,15 @@ class LLMHandler(ABC):
                 error_message = self.create_tool_message(error_call, error_response)
                 updated_messages.append(error_message)
 
-                call_parts = call.name.split("_")
-                if len(call_parts) >= 2:
-                    tool_id = call_parts[-1]  # Last part is tool ID (e.g., "1")
-                    action_name = "_".join(call_parts[:-1])
-                    tool_name = tools_dict.get(tool_id, {}).get("name", "unknown_tool")
-                    full_action_name = f"{action_name}_{tool_id}"
+                mapping = agent.tool_executor._name_to_tool
+                if call.name in mapping:
+                    resolved_tool_id, _ = mapping[call.name]
+                    tool_name = tools_dict.get(resolved_tool_id, {}).get(
+                        "name", "unknown_tool"
+                    )
                 else:
                     tool_name = "unknown_tool"
-                    action_name = call.name
-                    full_action_name = call.name
+                full_action_name = call.name
                 yield {
                     "type": "tool_call",
                     "data": {
@@ -823,7 +877,7 @@ class LLMHandler(ABC):
                         "status": "error",
                     },
                 }
-        return updated_messages
+        return updated_messages, pending_actions if pending_actions else None
 
     def handle_non_streaming(
         self, agent, response: Any, tools_dict: Dict, messages: List[Dict]
@@ -851,8 +905,22 @@ class LLMHandler(ABC):
                 try:
                     yield next(tool_handler_gen)
                 except StopIteration as e:
-                    messages = e.value
+                    messages, pending_actions = e.value
                     break
+
+            # If tools need approval or client execution, pause the loop
+            if pending_actions:
+                agent._pending_continuation = {
+                    "messages": messages,
+                    "pending_tool_calls": pending_actions,
+                    "tools_dict": tools_dict,
+                }
+                yield {
+                    "type": "tool_calls_pending",
+                    "data": {"pending_tool_calls": pending_actions},
+                }
+                return ""
+
             response = agent.llm.gen(
                 model=agent.model_id, messages=messages, tools=agent.tools
             )
@@ -913,10 +981,23 @@ class LLMHandler(ABC):
                     try:
                         yield next(tool_handler_gen)
                     except StopIteration as e:
-                        messages = e.value
+                        messages, pending_actions = e.value
                         break
                 tool_calls = {}
 
+                # If tools need approval or client execution, pause the loop
+                if pending_actions:
+                    agent._pending_continuation = {
+                        "messages": messages,
+                        "pending_tool_calls": pending_actions,
+                        "tools_dict": tools_dict,
+                    }
+                    yield {
+                        "type": "tool_calls_pending",
+                        "data": {"pending_tool_calls": pending_actions},
+                    }
+                    return
+
                 # Check if context limit was reached during tool execution
                 if hasattr(agent, 'context_limit_reached') and agent.context_limit_reached:
                     # Add system message warning about context limit

application/llm/handlers/google.py

@@ -67,18 +67,18 @@ class GoogleLLMHandler(LLMHandler):
             )
 
     def create_tool_message(self, tool_call: ToolCall, result: Any) -> Dict:
-        """Create Google-style tool message."""
+        """Create a tool result message in the standard internal format."""
+        import json as _json
 
+        content = (
+            _json.dumps(result)
+            if not isinstance(result, str)
+            else result
+        )
         return {
-            "role": "model",
-            "content": [
-                {
-                    "function_response": {
-                        "name": tool_call.name,
-                        "response": {"result": result},
-                    }
-                }
-            ],
+            "role": "tool",
+            "tool_call_id": tool_call.id,
+            "content": content,
         }
 
     def _iterate_stream(self, response: Any) -> Generator:

application/llm/handlers/handler_creator.py

@@ -7,6 +7,7 @@ class LLMHandlerCreator:
     handlers = {
         "openai": OpenAILLMHandler,
         "google": GoogleLLMHandler,
+        "novita": OpenAILLMHandler,  # Novita uses OpenAI-compatible API
         "default": OpenAILLMHandler,
     }
 

application/llm/handlers/openai.py

@@ -37,18 +37,18 @@ class OpenAILLMHandler(LLMHandler):
         )
 
     def create_tool_message(self, tool_call: ToolCall, result: Any) -> Dict:
-        """Create OpenAI-style tool message."""
+        """Create a tool result message in the standard internal format."""
+        import json as _json
+
+        content = (
+            _json.dumps(result)
+            if not isinstance(result, str)
+            else result
+        )
         return {
             "role": "tool",
-            "content": [
-                {
-                    "function_response": {
-                        "name": tool_call.name,
-                        "response": {"result": result},
-                        "call_id": tool_call.id,
-                    }
-                }
-            ],
+            "tool_call_id": tool_call.id,
+            "content": content,
         }
 
     def _iterate_stream(self, response: Any) -> Generator:

application/llm/novita.py

@@ -1,13 +1,13 @@
 from application.core.settings import settings
 from application.llm.openai import OpenAILLM
 
-NOVITA_BASE_URL = "https://api.novita.ai/v3/openai"
+NOVITA_BASE_URL = "https://api.novita.ai/openai"
 
 
 class NovitaLLM(OpenAILLM):
     def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs):
         super().__init__(
-            api_key=api_key or settings.API_KEY,
+            api_key=api_key or settings.NOVITA_API_KEY or settings.API_KEY,
             user_api_key=user_api_key,
             base_url=base_url or NOVITA_BASE_URL,
             *args,

application/llm/openai.py

@@ -91,16 +91,52 @@ class OpenAILLM(BaseLLM):
 
             if role == "model":
                 role = "assistant"
+
+            # Standard format: assistant message with tool_calls (passthrough)
+            tool_calls = message.get("tool_calls")
+            if tool_calls and role == "assistant":
+                cleaned_tcs = []
+                for tc in tool_calls:
+                    func = tc.get("function", {})
+                    args = func.get("arguments", "{}")
+                    if isinstance(args, dict):
+                        args = json.dumps(self._remove_null_values(args))
+                    elif isinstance(args, str):
+                        try:
+                            parsed = json.loads(args)
+                            args = json.dumps(self._remove_null_values(parsed))
+                        except (json.JSONDecodeError, TypeError):
+                            pass
+                    cleaned_tcs.append({
+                        "id": tc.get("id", ""),
+                        "type": "function",
+                        "function": {"name": func.get("name", ""), "arguments": args},
+                    })
+                cleaned_messages.append({
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": cleaned_tcs,
+                })
+                continue
+
+            # Standard format: tool message with tool_call_id (passthrough)
+            tool_call_id = message.get("tool_call_id")
+            if role == "tool" and tool_call_id is not None:
+                cleaned_messages.append({
+                    "role": "tool",
+                    "tool_call_id": tool_call_id,
+                    "content": content if isinstance(content, str) else json.dumps(content),
+                })
+                continue
+
             if role and content is not None:
                 if isinstance(content, str):
                     cleaned_messages.append({"role": role, "content": content})
                 elif isinstance(content, list):
-                    # Collect all content parts into a single message
                     content_parts = []
-
                     for item in content:
+                        # Legacy format support: function_call / function_response
                         if "function_call" in item:
-                            # Function calls need their own message
                             args = item["function_call"]["args"]
                             if isinstance(args, str):
                                 try:
@@ -116,28 +152,20 @@ class OpenAILLM(BaseLLM):
                                     "arguments": json.dumps(cleaned_args),
                                 },
                             }
-                            cleaned_messages.append(
-                                {
-                                    "role": "assistant",
-                                    "content": None,
-                                    "tool_calls": [tool_call],
-                                }
-                            )
+                            cleaned_messages.append({
+                                "role": "assistant",
+                                "content": None,
+                                "tool_calls": [tool_call],
+                            })
                         elif "function_response" in item:
-                            # Function responses need their own message
-                            cleaned_messages.append(
-                                {
-                                    "role": "tool",
-                                    "tool_call_id": item["function_response"][
-                                        "call_id"
-                                    ],
-                                    "content": json.dumps(
-                                        item["function_response"]["response"]["result"]
-                                    ),
-                                }
-                            )
+                            cleaned_messages.append({
+                                "role": "tool",
+                                "tool_call_id": item["function_response"]["call_id"],
+                                "content": json.dumps(
+                                    item["function_response"]["response"]["result"]
+                                ),
+                            })
                         elif isinstance(item, dict):
-                            # Collect content parts (text, images, files) into a single message
                             if "type" in item and item["type"] == "text" and "text" in item:
                                 content_parts.append(item)
                             elif "type" in item and item["type"] == "file" and "file" in item:
@@ -145,10 +173,7 @@ class OpenAILLM(BaseLLM):
                             elif "type" in item and item["type"] == "image_url" and "image_url" in item:
                                 content_parts.append(item)
                             elif "text" in item and "type" not in item:
-                                # Legacy format: {"text": "..."} without type
                                 content_parts.append({"type": "text", "text": item["text"]})
-
-                    # Add the collected content parts as a single message
                     if content_parts:
                         cleaned_messages.append({"role": role, "content": content_parts})
                 else:

application/parser/file/epub_parser.py

@@ -19,25 +19,10 @@ class EpubParser(BaseParser):
     def parse_file(self, file: Path, errors: str = "ignore") -> str:
         """Parse file."""
         try:
-            import ebooklib
-            from ebooklib import epub
+            from fast_ebook import epub
         except ImportError:
-            raise ValueError("`EbookLib` is required to read Epub files.")
-        try:
-            import html2text
-        except ImportError:
-            raise ValueError("`html2text` is required to parse Epub files.")
+            raise ValueError("`fast-ebook` is required to read Epub files.")
 
-        text_list = []
-        book = epub.read_epub(file, options={"ignore_ncx": True})
-
-        # Iterate through all chapters.
-        for item in book.get_items():
-            # Chapters are typically located in epub documents items.
-            if item.get_type() == ebooklib.ITEM_DOCUMENT:
-                text_list.append(
-                    html2text.html2text(item.get_content().decode("utf-8"))
-                )
-
-        text = "\n".join(text_list)
+        book = epub.read_epub(file)
+        text = book.to_markdown()
         return text

application/requirements.txt

@@ -1,9 +1,10 @@
-anthropic==0.75.0
-boto3==1.42.17
+alembic>=1.13,<2
+anthropic==0.88.0
+boto3==1.42.83
 beautifulsoup4==4.14.3
 cel-python==0.5.0
-celery==5.6.0
-cryptography==46.0.3
+celery==5.6.3
+cryptography==46.0.6
 dataclasses-json==0.6.7
 defusedxml==0.7.1
 docling>=2.16.0
@@ -11,89 +12,84 @@ rapidocr>=1.4.0
 onnxruntime>=1.19.0
 docx2txt==0.9
 ddgs>=8.0.0
-ebooklib==0.20
-escodegen==1.0.11
-esprima==4.0.1
-esutils==1.0.1
-elevenlabs==2.27.0
-Flask==3.1.2
+fast-ebook
+elevenlabs==2.41.0
+Flask==3.1.3
 faiss-cpu==1.13.2
-fastmcp==2.14.1
+fastmcp==3.2.0
 flask-restx==1.3.2
-google-genai==1.54.0
-google-api-python-client==2.187.0
-google-auth-httplib2==0.3.0
-google-auth-oauthlib==1.2.3
+google-genai==1.69.0
+google-api-python-client==2.193.0
+google-auth-httplib2==0.3.1
+google-auth-oauthlib==1.3.1
 gTTS==2.5.4
-gunicorn==23.0.0
-html2text==2025.4.15
-javalang==0.13.0
+gunicorn==25.3.0
 jinja2==3.1.6
-jiter==0.12.0
-jmespath==1.0.1
+jiter==0.13.0
+jmespath==1.1.0
 joblib==1.5.3
 jsonpatch==1.33
 jsonpointer==3.0.0
-kombu==5.6.1
-langchain==1.2.0
+kombu==5.6.2
+langchain==1.2.3
 langchain-community==0.4.1
-langchain-core==1.2.5
-langchain-openai==1.1.6
-langchain-text-splitters==1.1.0
-langsmith==0.5.1
+langchain-core==1.2.23
+langchain-openai==1.1.12
+langchain-text-splitters==1.1.1
+langsmith==0.7.23
 lazy-object-proxy==1.12.0
 lxml==6.0.2
 markupsafe==3.0.3
 marshmallow>=3.18.0,<5.0.0
 mpmath==1.3.0
-multidict==6.7.0
-msal==1.34.0
+multidict==6.7.1
+msal==1.35.1
 mypy-extensions==1.1.0
 networkx==3.6.1
-numpy==2.4.0
-openai==2.14.0
+numpy==2.4.4
+openai==2.30.0
 openapi3-parser==1.1.22
-orjson==3.11.5
-packaging==24.2
-pandas==2.3.3
+orjson==3.11.7
+packaging==26.0
+pandas==3.0.2
 openpyxl==3.1.5
-pathable==0.4.4
+pathable==0.5.0
 pdf2image>=1.17.0
 pillow
-portalocker>=2.7.0,<3.0.0
-prance==25.4.8.0
+portalocker>=2.7.0,<4.0.0
 prompt-toolkit==3.0.52
-protobuf==6.33.2
-psycopg2-binary==2.9.11
+protobuf==7.34.1
+psycopg[binary,pool]>=3.1,<4
 py==1.11.0
 pydantic
 pydantic-core
 pydantic-settings
-pymongo==4.15.5
-pypdf==6.5.0
+pymongo==4.16.0
+pypdf==6.9.2
 python-dateutil==2.9.0.post0
 python-dotenv
 python-jose==3.5.0
 python-pptx==1.0.2
-redis==7.1.0
+redis==7.4.0
 referencing>=0.28.0,<0.38.0
-regex==2025.11.3
-requests==2.32.5
+regex==2026.4.4
+requests==2.33.1
 retry==0.9.2
-sentence-transformers==5.2.0
+sentence-transformers==5.3.0
+sqlalchemy>=2.0,<3
 tiktoken==0.12.0
-tokenizers==0.22.1
-torch==2.9.1
-tqdm==4.67.1
-transformers==4.57.3
+tokenizers==0.22.2
+torch==2.11.0
+tqdm==4.67.3
+transformers==5.4.0
 typing-extensions==4.15.0
 typing-inspect==0.9.0
 tzdata==2025.3
 urllib3==2.6.3
 vine==5.1.0
-wcwidth==0.2.14
+wcwidth==0.6.0
 werkzeug>=3.1.0
-yarl==1.22.0
+yarl==1.23.0
 markdownify==1.2.2
-tldextract==5.3.0
-websockets==15.0.1
+tldextract==5.3.1
+websockets==16.0

application/storage/db/__init__.py

@@ -0,0 +1,10 @@
+"""PostgreSQL storage layer for user-level data.
+
+This package holds the SQLAlchemy Core engine, metadata, repositories, and
+migration infrastructure for the user-data Postgres database. It is separate
+from ``application/vectorstore/pgvector.py`` — the two may point at the same
+cluster or at different clusters depending on operator configuration.
+
+Repository modules are added in later phases
+as individual collections are ported.
+"""

application/storage/db/base_repository.py

@@ -0,0 +1,39 @@
+"""Common helpers shared by all repositories.
+
+Repositories are thin wrappers around SQLAlchemy Core query construction.
+They take a ``Connection`` on call and return plain ``dict`` rows during the
+Mongo→Postgres cutover so that call sites don't have to change shape. Once
+cutover is complete, a follow-up phase may migrate repo return types to
+Pydantic DTOs (tracked in the migration plan as a post-migration item).
+"""
+
+from typing import Any, Mapping
+from uuid import UUID
+
+
+def row_to_dict(row: Any) -> dict:
+    """Convert a SQLAlchemy ``Row`` to a plain dict with Mongo-compatible ids.
+
+    During the migration window, API responses and downstream code still
+    expect a string ``_id`` field (matching the Mongo shape). This helper
+    normalizes UUID columns to strings and emits both ``id`` and ``_id`` so
+    existing serializers keep working unchanged.
+
+    Args:
+        row: A SQLAlchemy ``Row`` object, or ``None``.
+
+    Returns:
+        A plain dict, or an empty dict if ``row`` is ``None``.
+    """
+    if row is None:
+        return {}
+
+    # Row has a ``._mapping`` attribute exposing a MappingProxy view.
+    mapping: Mapping[str, Any] = row._mapping  # type: ignore[attr-defined]
+    out = dict(mapping)
+
+    if "id" in out and out["id"] is not None:
+        out["id"] = str(out["id"]) if isinstance(out["id"], UUID) else out["id"]
+        out["_id"] = out["id"]
+
+    return out

application/storage/db/dual_write.py

@@ -0,0 +1,67 @@
+"""Best-effort Postgres dual-write helper used during the MongoDB→Postgres
+migration.
+
+The helper:
+
+* Returns immediately if ``settings.USE_POSTGRES`` is off, so default-off
+  call sites add literally zero work.
+* Opens a transactional connection from the user-data SQLAlchemy engine.
+* Instantiates the caller's repository class on that connection.
+* Runs the caller's operation.
+* Swallows and logs any exception. **Mongo remains the source of truth
+  during the dual-write window** — a Postgres-side failure must never
+  break a user-facing request. Drift that builds up from swallowed
+  failures is caught separately by re-running the backfill script.
+
+Call sites look like::
+
+    users_collection.update_one(..., {"$addToSet": {...}})             # Mongo write, unchanged
+    dual_write(UsersRepository, lambda r: r.add_pinned(uid, aid))      # Postgres mirror
+
+A single parameterised helper rather than one function per collection
+means a new collection just needs its repository class — no new helper
+function, no new feature flag. The whole helper is deleted at Phase 5
+when the migration is complete.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Callable, TypeVar
+
+from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+_Repo = TypeVar("_Repo")
+
+
+def dual_write(repo_cls: type[_Repo], fn: Callable[[_Repo], None]) -> None:
+    """Mirror a Mongo write into Postgres via ``repo_cls``, best-effort.
+
+    No-op when ``settings.USE_POSTGRES`` is false. Any exception
+    (connection pool exhaustion, migration drift, SQL error) is logged
+    and swallowed so the caller's primary Mongo write remains the source
+    of truth.
+
+    Args:
+        repo_cls: The repository class to instantiate (e.g. ``UsersRepository``).
+        fn: A callable that takes the instantiated repository and performs
+            the desired write.
+    """
+    if not settings.USE_POSTGRES:
+        return
+
+    try:
+        # Lazy import so modules that import dual_write don't pay the
+        # SQLAlchemy import cost when the flag is off.
+        from application.storage.db.engine import get_engine
+
+        with get_engine().begin() as conn:
+            fn(repo_cls(conn))
+    except Exception:
+        logger.warning(
+            "Postgres dual-write failed for %s — Mongo write already committed",
+            repo_cls.__name__,
+            exc_info=True,
+        )

application/storage/db/engine.py

@@ -0,0 +1,67 @@
+"""SQLAlchemy Core engine factory for the user-data Postgres database.
+
+The engine is lazily constructed on first use and cached as a module-level
+singleton. Repositories and the Alembic env module both obtain connections
+through this factory, so pool tuning lives in one place.
+
+``POSTGRES_URI`` can be written in any of the common Postgres URI forms::
+
+    postgres://user:pass@host:5432/docsgpt
+    postgresql://user:pass@host:5432/docsgpt
+
+Both are accepted and normalized internally to the psycopg3 dialect
+(``postgresql+psycopg://``) by ``application.core.settings``. Operators
+don't need to know about SQLAlchemy dialect prefixes.
+"""
+
+from typing import Optional
+
+from sqlalchemy import Engine, create_engine
+
+from application.core.settings import settings
+
+_engine: Optional[Engine] = None
+
+
+def get_engine() -> Engine:
+    """Return the process-wide SQLAlchemy Engine, creating it if needed.
+
+    Raises:
+        RuntimeError: If ``settings.POSTGRES_URI`` is unset. Callers that
+            reach this path without a configured URI have a setup bug — the
+            error message points them at the right setting.
+
+    Returns:
+        A SQLAlchemy ``Engine`` configured with a pooled connection to
+        Postgres via psycopg3.
+    """
+    global _engine
+    if _engine is None:
+        if not settings.POSTGRES_URI:
+            raise RuntimeError(
+                "POSTGRES_URI is not configured. Set it in your .env to a "
+                "psycopg3 URI such as "
+                "'postgresql+psycopg://user:pass@host:5432/docsgpt'."
+            )
+        _engine = create_engine(
+            settings.POSTGRES_URI,
+            pool_size=10,
+            max_overflow=20,
+            pool_pre_ping=True,     # survive PgBouncer / idle-disconnect recycles
+            pool_recycle=1800,
+            future=True,
+        )
+    return _engine
+
+
+def dispose_engine() -> None:
+    """Dispose the pooled connections and reset the singleton.
+
+    Called from the Celery ``worker_process_init`` signal so each forked
+    worker gets a fresh pool instead of sharing file descriptors with the
+    parent process (which corrupts the pool on fork).
+    """
+    global _engine
+    if _engine is not None:
+        _engine.dispose()
+        _engine = None

application/storage/db/models.py

@@ -0,0 +1,38 @@
+"""SQLAlchemy Core metadata for the user-data Postgres database.
+
+Tables are added here one at a time as repositories are built during the
+MongoDB→Postgres migration. The baseline schema in the Alembic migration
+(``application/alembic/versions/0001_initial.py``) is the source of truth
+for DDL; the ``Table`` definitions below must match it column-for-column.
+If the two drift, migrations win — update this file to match.
+"""
+
+from sqlalchemy import (
+    Column,
+    DateTime,
+    MetaData,
+    Table,
+    Text,
+    func,
+)
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+
+metadata = MetaData()
+
+
+# --- Phase 1, Tier 1 --------------------------------------------------------
+
+users_table = Table(
+    "users",
+    metadata,
+    Column("id", UUID(as_uuid=True), primary_key=True, server_default=func.gen_random_uuid()),
+    Column("user_id", Text, nullable=False, unique=True),
+    Column(
+        "agent_preferences",
+        JSONB,
+        nullable=False,
+        server_default='{"pinned": [], "shared_with_me": []}',
+    ),
+    Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
+    Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
+)

application/storage/db/repositories/__init__.py

@@ -0,0 +1,11 @@
+"""Repositories for the user-data Postgres database.
+
+Each module in this package exposes exactly one repository class. Repository
+methods take a ``Connection`` (either as a constructor argument or as a
+method argument) and return plain ``dict`` rows via
+``application.storage.db.base_repository.row_to_dict`` during the
+MongoDB→Postgres cutover, so call sites don't have to change shape.
+
+Repositories are added one collection at a time, matching the phased
+rollout in ``migration-postgres.md``.
+"""

application/storage/db/repositories/users.py

@@ -0,0 +1,245 @@
+"""Repository for the ``users`` table.
+
+Covers every operation the legacy Mongo code performs on
+``users_collection``:
+
+1. ``ensure_user_doc`` in ``application/api/user/base.py`` (upsert + get)
+2. Pin/unpin agents in ``application/api/user/agents/routes.py`` (add/remove
+   on ``agent_preferences.pinned``)
+3. Share accept/reject in ``application/api/user/agents/sharing.py`` (add/
+   bulk-remove on ``agent_preferences.shared_with_me``)
+4. Cascade delete of an agent id from both arrays at once
+
+All array mutations are implemented as single atomic UPDATE statements
+using JSONB operators (``jsonb_set``, ``jsonb_array_elements``, ``@>``)
+so there is no read-modify-write race between concurrent writers on the
+same user row.
+
+The repository takes a ``Connection`` and does not manage its own
+transactions. Callers are responsible for wrapping writes in
+``with engine.begin() as conn:`` (production) or the test fixture's
+rollback-per-test connection (tests).
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+from sqlalchemy import Connection, text
+
+from application.storage.db.base_repository import row_to_dict
+
+
+_DEFAULT_PREFERENCES = '{"pinned": [], "shared_with_me": []}'
+
+
+class UsersRepository:
+    """Postgres-backed replacement for Mongo ``users_collection`` writes/reads."""
+
+    def __init__(self, conn: Connection) -> None:
+        self._conn = conn
+
+    # ------------------------------------------------------------------
+    # Reads
+    # ------------------------------------------------------------------
+    def get(self, user_id: str) -> Optional[dict]:
+        """Return the user row as a dict, or ``None`` if missing.
+
+        Args:
+            user_id: Auth-provider ``sub`` (opaque string).
+        """
+        result = self._conn.execute(
+            text("SELECT * FROM users WHERE user_id = :user_id"),
+            {"user_id": user_id},
+        )
+        row = result.fetchone()
+        return row_to_dict(row) if row is not None else None
+
+    # ------------------------------------------------------------------
+    # Upsert
+    # ------------------------------------------------------------------
+    def upsert(self, user_id: str) -> dict:
+        """Ensure a row exists for ``user_id`` and return it.
+
+        Matches Mongo's ``find_one_and_update(..., $setOnInsert, upsert=True,
+        return_document=AFTER)`` semantics: if the row exists, preferences
+        are preserved untouched; if it doesn't, a new row is created with
+        default preferences.
+
+        The ``DO UPDATE SET user_id = EXCLUDED.user_id`` branch is a
+        deliberate no-op that lets ``RETURNING *`` fire on both the insert
+        and conflict paths (``DO NOTHING`` would suppress the returning).
+        """
+        result = self._conn.execute(
+            text(
+                """
+                INSERT INTO users (user_id, agent_preferences)
+                VALUES (:user_id, CAST(:default_prefs AS jsonb))
+                ON CONFLICT (user_id) DO UPDATE
+                    SET user_id = EXCLUDED.user_id
+                RETURNING *
+                """
+            ),
+            {"user_id": user_id, "default_prefs": _DEFAULT_PREFERENCES},
+        )
+        return row_to_dict(result.fetchone())
+
+    # ------------------------------------------------------------------
+    # Pinned agents
+    # ------------------------------------------------------------------
+    def add_pinned(self, user_id: str, agent_id: str) -> None:
+        """Idempotently append ``agent_id`` to ``agent_preferences.pinned``.
+
+        Uses ``@>`` containment so a duplicate add is a no-op rather than a
+        silent double-insert. The whole update is a single atomic statement
+        so concurrent add_pinned calls on the same user cannot interleave
+        into a read-modify-write race.
+        """
+        self._append_to_jsonb_array(user_id, "pinned", agent_id)
+
+    def remove_pinned(self, user_id: str, agent_id: str) -> None:
+        """Remove ``agent_id`` from ``agent_preferences.pinned`` if present."""
+        self._remove_from_jsonb_array(user_id, "pinned", [agent_id])
+
+    def remove_pinned_bulk(self, user_id: str, agent_ids: Iterable[str]) -> None:
+        """Remove every id in ``agent_ids`` from ``agent_preferences.pinned``.
+
+        No-op if the list is empty. Unknown ids are silently ignored so
+        callers can pass the full "stale" set without pre-filtering.
+        """
+        ids = list(agent_ids)
+        if not ids:
+            return
+        self._remove_from_jsonb_array(user_id, "pinned", ids)
+
+    # ------------------------------------------------------------------
+    # Shared-with-me agents
+    # ------------------------------------------------------------------
+    def add_shared(self, user_id: str, agent_id: str) -> None:
+        """Idempotently append ``agent_id`` to ``agent_preferences.shared_with_me``."""
+        self._append_to_jsonb_array(user_id, "shared_with_me", agent_id)
+
+    def remove_shared_bulk(self, user_id: str, agent_ids: Iterable[str]) -> None:
+        """Bulk-remove from ``agent_preferences.shared_with_me``. Empty list is a no-op."""
+        ids = list(agent_ids)
+        if not ids:
+            return
+        self._remove_from_jsonb_array(user_id, "shared_with_me", ids)
+
+    # ------------------------------------------------------------------
+    # Combined removal — called when an agent is hard-deleted
+    # ------------------------------------------------------------------
+    def remove_agent_from_all(self, user_id: str, agent_id: str) -> None:
+        """Remove ``agent_id`` from BOTH pinned and shared_with_me atomically.
+
+        Mirrors the Mongo ``$pull`` that targets both nested array fields
+        in one ``update_one`` — see ``application/api/user/agents/routes.py``
+        around the agent-delete path.
+        """
+        self._conn.execute(
+            text(
+                """
+                UPDATE users
+                SET
+                    agent_preferences = jsonb_set(
+                        jsonb_set(
+                            agent_preferences,
+                            '{pinned}',
+                            COALESCE(
+                                (
+                                    SELECT jsonb_agg(elem)
+                                    FROM jsonb_array_elements(
+                                        COALESCE(agent_preferences->'pinned', '[]'::jsonb)
+                                    ) AS elem
+                                    WHERE (elem #>> '{}') != :agent_id
+                                ),
+                                '[]'::jsonb
+                            )
+                        ),
+                        '{shared_with_me}',
+                        COALESCE(
+                            (
+                                SELECT jsonb_agg(elem)
+                                FROM jsonb_array_elements(
+                                    COALESCE(agent_preferences->'shared_with_me', '[]'::jsonb)
+                                ) AS elem
+                                WHERE (elem #>> '{}') != :agent_id
+                            ),
+                            '[]'::jsonb
+                        )
+                    ),
+                    updated_at = now()
+                WHERE user_id = :user_id
+                """
+            ),
+            {"user_id": user_id, "agent_id": agent_id},
+        )
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+    def _append_to_jsonb_array(self, user_id: str, key: str, agent_id: str) -> None:
+        """Idempotent append of ``agent_id`` to ``agent_preferences.<key>``.
+
+        The ``key`` argument is NOT user input — it's hard-coded by the
+        calling method (``pinned`` / ``shared_with_me``). It goes into the
+        SQL literal because ``jsonb_set`` requires a path literal, not a
+        bind parameter. This is safe as long as callers never pass
+        untrusted strings for ``key``.
+        """
+        if key not in ("pinned", "shared_with_me"):
+            raise ValueError(f"unsupported jsonb key: {key!r}")
+        self._conn.execute(
+            text(
+                f"""
+                UPDATE users
+                SET
+                    agent_preferences = jsonb_set(
+                        agent_preferences,
+                        '{{{key}}}',
+                        CASE
+                            WHEN agent_preferences->'{key}' @> to_jsonb(CAST(:agent_id AS text))
+                                THEN agent_preferences->'{key}'
+                            ELSE
+                                COALESCE(agent_preferences->'{key}', '[]'::jsonb)
+                                || to_jsonb(CAST(:agent_id AS text))
+                        END
+                    ),
+                    updated_at = now()
+                WHERE user_id = :user_id
+                """
+            ),
+            {"user_id": user_id, "agent_id": agent_id},
+        )
+
+    def _remove_from_jsonb_array(
+        self, user_id: str, key: str, agent_ids: list[str]
+    ) -> None:
+        """Remove every id in ``agent_ids`` from ``agent_preferences.<key>``."""
+        if key not in ("pinned", "shared_with_me"):
+            raise ValueError(f"unsupported jsonb key: {key!r}")
+        self._conn.execute(
+            text(
+                f"""
+                UPDATE users
+                SET
+                    agent_preferences = jsonb_set(
+                        agent_preferences,
+                        '{{{key}}}',
+                        COALESCE(
+                            (
+                                SELECT jsonb_agg(elem)
+                                FROM jsonb_array_elements(
+                                    COALESCE(agent_preferences->'{key}', '[]'::jsonb)
+                                ) AS elem
+                                WHERE NOT ((elem #>> '{{}}') = ANY(:agent_ids))
+                            ),
+                            '[]'::jsonb
+                        )
+                    ),
+                    updated_at = now()
+                WHERE user_id = :user_id
+                """
+            ),
+            {"user_id": user_id, "agent_ids": agent_ids},
+        )

application/storage/local.py

@@ -21,10 +21,19 @@ class LocalStorage(BaseStorage):
         )
 
     def _get_full_path(self, path: str) -> str:
-        """Get absolute path by combining base_dir and path."""
+        """Get absolute path by combining base_dir and path.
+
+        Raises:
+            ValueError: If the resolved path escapes base_dir (path traversal).
+        """
         if os.path.isabs(path):
-            return path
-        return os.path.join(self.base_dir, path)
+            resolved = os.path.realpath(path)
+        else:
+            resolved = os.path.realpath(os.path.join(self.base_dir, path))
+        base = os.path.realpath(self.base_dir)
+        if not resolved.startswith(base + os.sep) and resolved != base:
+            raise ValueError(f"Path traversal detected: {path}")
+        return resolved
 
     def save_file(self, file_data: BinaryIO, path: str, **kwargs) -> dict:
         """Save a file to local storage."""

application/storage/s3.py

@@ -2,6 +2,7 @@
 
 import io
 import os
+import posixpath
 from typing import BinaryIO, Callable, List
 
 import boto3
@@ -14,6 +15,20 @@ from botocore.exceptions import ClientError
 class S3Storage(BaseStorage):
     """AWS S3 storage implementation."""
 
+    @staticmethod
+    def _validate_path(path: str) -> str:
+        """Validate and normalize an S3 key to prevent path traversal.
+
+        Raises:
+            ValueError: If the path contains traversal sequences or is absolute.
+        """
+        if "\x00" in path:
+            raise ValueError(f"Null byte in path: {path}")
+        normalized = posixpath.normpath(path)
+        if normalized.startswith("/") or normalized.startswith(".."):
+            raise ValueError(f"Path traversal detected: {path}")
+        return normalized
+
     def __init__(self, bucket_name=None):
         """
         Initialize S3 storage.
@@ -46,6 +61,7 @@ class S3Storage(BaseStorage):
         **kwargs,
     ) -> dict:
         """Save a file to S3 storage."""
+        path = self._validate_path(path)
         self.s3.upload_fileobj(
             file_data, self.bucket_name, path, ExtraArgs={"StorageClass": storage_class}
         )
@@ -61,6 +77,7 @@ class S3Storage(BaseStorage):
 
     def get_file(self, path: str) -> BinaryIO:
         """Get a file from S3 storage."""
+        path = self._validate_path(path)
         if not self.file_exists(path):
             raise FileNotFoundError(f"File not found: {path}")
         file_obj = io.BytesIO()
@@ -70,6 +87,7 @@ class S3Storage(BaseStorage):
 
     def delete_file(self, path: str) -> bool:
         """Delete a file from S3 storage."""
+        path = self._validate_path(path)
         try:
             self.s3.delete_object(Bucket=self.bucket_name, Key=path)
             return True
@@ -78,6 +96,7 @@ class S3Storage(BaseStorage):
 
     def file_exists(self, path: str) -> bool:
         """Check if a file exists in S3 storage."""
+        path = self._validate_path(path)
         try:
             self.s3.head_object(Bucket=self.bucket_name, Key=path)
             return True
@@ -115,6 +134,7 @@ class S3Storage(BaseStorage):
         import logging
         import tempfile
 
+        path = self._validate_path(path)
         if not self.file_exists(path):
             raise FileNotFoundError(f"File not found in S3: {path}")
         with tempfile.NamedTemporaryFile(

application/vectorstore/faiss.py

@@ -11,11 +11,33 @@ from application.storage.storage_creator import StorageCreator
 
 
 def get_vectorstore(path: str) -> str:
-    if path:
-        vectorstore = f"indexes/{path}"
-    else:
-        vectorstore = "indexes"
-    return vectorstore
+    """Build a safe local path for a FAISS index.
+
+    Args:
+        path: Source identifier provided by the caller.
+
+    Returns:
+        The validated vectorstore path rooted under ``indexes``.
+
+    Raises:
+        ValueError: If ``path`` escapes the ``indexes`` directory.
+    """
+    base_dir = "indexes"
+    if not path:
+        return base_dir
+
+    normalized = str(path).strip()
+    if "\\" in normalized:
+        raise ValueError("Invalid source_id path")
+
+    candidate = os.path.normpath(os.path.join(base_dir, normalized))
+    base_abs = os.path.abspath(base_dir)
+    candidate_abs = os.path.abspath(candidate)
+
+    if not candidate_abs.startswith(base_abs + os.sep) and candidate_abs != base_abs:
+        raise ValueError("Invalid source_id path")
+
+    return candidate
 
 
 class FaissStore(BaseVectorStore):

application/vectorstore/pgvector.py

@@ -37,27 +37,25 @@ class PGVectorStore(BaseVectorStore):
             )
 
         try:
-            import psycopg2
-            from psycopg2.extras import Json
-            import pgvector.psycopg2
+            import psycopg
+            from pgvector.psycopg import register_vector
         except ImportError:
             raise ImportError(
                 "Could not import required packages. "
-                "Please install with `pip install psycopg2-binary pgvector`."
+                "Please install with `pip install 'psycopg[binary,pool]' pgvector`."
             )
 
-        self._psycopg2 = psycopg2
-        self._Json = Json
-        self._pgvector = pgvector.psycopg2
+        self._psycopg = psycopg
+        self._register_vector = register_vector
         self._connection = None
         self._ensure_table_exists()
 
     def _get_connection(self):
         """Get or create database connection"""
         if self._connection is None or self._connection.closed:
-            self._connection = self._psycopg2.connect(self._connection_string)
+            self._connection = self._psycopg.connect(self._connection_string)
             # Register pgvector types
-            self._pgvector.register_vector(self._connection)
+            self._register_vector(self._connection)
         return self._connection
 
     def _ensure_table_exists(self):
@@ -170,7 +168,7 @@ class PGVectorStore(BaseVectorStore):
             for text, embedding, metadata in zip(texts, embeddings, metadatas):
                 cursor.execute(
                     insert_query,
-                    (text, embedding, self._Json(metadata), self._source_id)
+                    (text, embedding, metadata, self._source_id)
                 )
                 inserted_id = cursor.fetchone()[0]
                 inserted_ids.append(str(inserted_id))
@@ -261,7 +259,7 @@ class PGVectorStore(BaseVectorStore):
             
             cursor.execute(
                 insert_query,
-                (text, embeddings[0], self._Json(final_metadata), self._source_id)
+                (text, embeddings[0], final_metadata, self._source_id)
             )
             inserted_id = cursor.fetchone()[0]
             conn.commit()

docs/content/Agents/_meta.js

@@ -7,6 +7,10 @@ export default {
     "title": "🔌 Agent API",
     "href": "/Agents/api"
   },
+  "openai-compatible": {
+    "title": "🔄 OpenAI-Compatible API",
+    "href": "/Agents/openai-compatible"
+  },
   "webhooks": {
     "title": "🪝 Agent Webhooks",
     "href": "/Agents/webhooks"

docs/content/Agents/api.mdx

@@ -15,6 +15,10 @@ DocsGPT Agents can be accessed programmatically through API endpoints. This page
 
 When you use an agent `api_key`, DocsGPT loads that agent's configuration automatically (prompt, tools, sources, default model). You usually only need to send `question` and `api_key`.
 
+<Callout type="info">
+Looking to connect an existing OpenAI-compatible client (opencode, aider, the OpenAI SDKs, etc.) to a DocsGPT Agent? Use the [OpenAI-Compatible Chat Completions API](/Agents/openai-compatible) — it speaks the standard chat completions protocol so no adapter code is required.
+</Callout>
+
 ## Base URL
 
 <Callout type="info">

docs/content/Agents/basics.mdx

@@ -44,36 +44,40 @@ The main set of instructions or system [prompt](/Guides/Customising-prompts) tha
 
 ## Understanding Agent Types
 
-DocsGPT allows for different "types" of agents, each with a distinct way of processing information and generating responses. The code for these agent types can be found in the `application/agents/` directory.
+DocsGPT supports several agent types, each with a distinct way of processing information. The code for these can be found in the `application/agents/` directory.
 
-### 1. Classic Agent (`classic_agent.py`)
+### 1. Classic Agent
 
-**How it works:** The Classic Agent follows a traditional Retrieval Augmented Generation (RAG) approach.
-    1.  **Retrieve:** When a query is made, it first searches the selected Source documents for relevant information.
-    2.  **Augment:** This retrieved data is then added to the context, along with the main Prompt and the user's query.
-    3.  **Generate:** The LLM generates a response based on this augmented context. It can also utilize any configured tools if the LLM decides they are necessary.
+The Classic Agent follows a traditional Retrieval Augmented Generation (RAG) approach: it retrieves relevant document chunks, augments the prompt context with them, and generates a response. It can also use configured tools if the LLM decides they are necessary.
 
-**Best for:**
-    * Direct question-answering over a specific set of documents.
-    * Tasks where the primary goal is to extract and synthesize information from the provided sources.
-    * Simpler tool integrations where the decision to use a tool is straightforward.
+**Best for:** Direct question-answering over a specific set of documents and straightforward tool use.
 
-### 2. ReAct Agent (`react_agent.py`)
+### 2. Agentic Agent
 
-**How it works:** The ReAct Agent employs a more sophisticated "Reason and Act" framework. This involves a multi-step process:
-    1.  **Plan (Thought):** Based on the query, its prompt, and available tools/sources, the LLM first generates a plan or a sequence of thoughts on how to approach the problem. You might see this output as a "thought" process during generation.
-    2.  **Act:** The agent then executes actions based on this plan. This might involve querying its sources, using a tool, or performing internal reasoning.
-    3.  **Observe:** It gathers observations from the results of its actions (e.g., data from a tool, snippets from documents).
-    4.  **Repeat (if necessary):** Steps 2 and 3 can be repeated as the agent refines its approach or gathers more information.
-    5.  **Conclude:** Finally, it generates the final answer based on the initial query and all accumulated observations.
+Unlike Classic which pre-fetches documents into the prompt, the Agentic Agent gives the LLM an `internal_search` tool so it can decide **when, what, and whether** to search. This means the LLM controls its own retrieval — it can search multiple times, refine queries, or skip retrieval entirely if the question doesn't need it.
 
-**Best for:**
-    * More complex tasks that require multi-step reasoning or problem-solving.
-    * Scenarios where the agent needs to dynamically decide which tools to use and in what order, based on intermediate results.
-    * Interactive tasks where the agent needs to "think" through a problem.
+**Best for:** Tasks where the agent needs to dynamically decide how to gather information, use multiple tools in sequence, or combine retrieval with external tool calls.
+
+### 3. Research Agent
+
+A multi-phase agent designed for in-depth research tasks:
+1. **Clarification** — Determines if the question needs clarification before proceeding.
+2. **Planning** — Decomposes the question into research steps with adaptive depth based on complexity.
+3. **Research** — Executes each step, calling tools and refining queries as needed.
+4. **Synthesis** — Compiles findings into a final cited report.
+
+Includes budget controls for max steps, timeout, and token limits to keep research bounded.
+
+**Best for:** Complex questions that require multi-step investigation, gathering information from multiple sources, and producing structured reports with citations.
+
+### 4. Workflow Agent
+
+Executes predefined workflows composed of connected nodes (AI Agent, Set State, Condition). See the [Workflow Nodes](/Agents/nodes) page for details on building workflows.
+
+**Best for:** Structured, multi-step processes with branching logic and shared state between steps.
 
 <Callout type="info">
-Developers looking to introduce new agent architectures can explore the `application/agents/` directory. `classic_agent.py` and `react_agent.py` serve as excellent starting points, demonstrating how to inherit from `BaseAgent` and structure agent logic.
+The legacy "ReAct" agent type is still accepted for backwards compatibility but maps to the Classic Agent internally. New agents should use Classic, Agentic, or Research instead.
 </Callout>
 
 ## Navigating and Managing Agents in DocsGPT
@@ -107,6 +111,7 @@ Once an agent is created, you can:
     * Modify any of its configuration settings (name, description, source, prompt, tools, type).
     * **Generate a Public Link:** From the edit screen, you can create a shareable public link that allows others to import and use your agent.
     * **Get a Webhook URL:** You can also obtain a Webhook URL for the agent. This allows external applications or services to trigger the agent and receive responses programmatically, enabling powerful integrations and automations.
+* **Use it via API:** Every agent exposes an API key that can be used with the native [Agent API](/Agents/api) or the [OpenAI-Compatible API](/Agents/openai-compatible) so you can drop DocsGPT Agents into any tool that already speaks the chat completions protocol.
 
 ## Seeding Premade Agents from YAML
 

docs/content/Agents/openai-compatible.mdx

@@ -0,0 +1,93 @@
+---
+title: OpenAI-Compatible API
+description: Connect any OpenAI-compatible client to DocsGPT Agents via /v1/chat/completions.
+---
+
+import { Callout, Tabs } from 'nextra/components';
+
+# OpenAI-Compatible API
+
+DocsGPT exposes `/v1/chat/completions` following the standard chat completions protocol. Point any compatible client — **opencode**, **Aider**, **LibreChat** or the OpenAI SDKs — at your DocsGPT Agent by changing only the base URL and API key.
+
+## Quick Start
+
+<Tabs items={['Python', 'cURL']}>
+  <Tabs.Tab>
+    ```python
+    from openai import OpenAI
+
+    client = OpenAI(
+        base_url="http://localhost:7091/v1",   # or https://gptcloud.arc53.com/v1
+        api_key="your_agent_api_key",
+    )
+
+    response = client.chat.completions.create(
+        model="docsgpt-agent",
+        messages=[{"role": "user", "content": "Summarize our refund policy"}],
+    )
+    print(response.choices[0].message.content)
+    ```
+  </Tabs.Tab>
+  <Tabs.Tab>
+    ```bash
+    curl -X POST http://localhost:7091/v1/chat/completions \
+      -H "Authorization: Bearer your_agent_api_key" \
+      -H "Content-Type: application/json" \
+      -d '{"model":"docsgpt-agent","messages":[{"role":"user","content":"Summarize our refund policy"}]}'
+    ```
+  </Tabs.Tab>
+</Tabs>
+
+The `model` field is accepted but ignored — the agent bound to your API key determines the model. The agent's prompt, sources, tools, and default model are loaded automatically.
+
+## Base URL & Auth
+
+| Environment | Base URL |
+| --- | --- |
+| Local | `http://localhost:7091/v1` |
+| Cloud | `https://gptcloud.arc53.com/v1` |
+
+Authenticate with `Authorization: Bearer <agent_api_key>`.
+
+## Endpoints
+
+| Method | Path | Description |
+| --- | --- | --- |
+| `POST` | `/v1/chat/completions` | Chat request (streaming or non-streaming) |
+| `GET` | `/v1/models` | List agents available to your key |
+
+## Streaming
+
+Set `"stream": true`. You'll receive SSE chunks with `choices[0].delta.content`. DocsGPT-specific events (sources, tool calls) arrive as extra frames with a `docsgpt` key — standard clients ignore them.
+
+```python
+stream = client.chat.completions.create(
+    model="docsgpt-agent",
+    stream=True,
+    messages=[{"role": "user", "content": "Explain vector search"}],
+)
+for chunk in stream:
+    print(chunk.choices[0].delta.content or "", end="", flush=True)
+```
+
+## System Prompt Override
+
+System messages are **dropped by default** — the agent's configured prompt is used. To allow callers to override it, enable **Allow prompt override** in the agent's Advanced settings.
+
+<Callout type="warning">
+When an override is active, the agent's prompt template is replaced wholesale — template variables like `{summaries}` are not substituted.
+</Callout>
+
+## Conversation Persistence
+
+Conversations are **not persisted by default** (stateless, like most OpenAI clients expect). Opt in per request:
+
+```json
+{ "docsgpt": { "save_conversation": true } }
+```
+
+The response will include `docsgpt.conversation_id`.
+
+## When to Use Native Endpoints Instead
+
+Use [`/api/answer` or `/stream`](/Agents/api) if you need server-side attachments, `passthrough` template variables, explicit `conversation_id` reuse, or persistence by default.

docs/content/Deploying/Amazon-Lightsail.mdx

@@ -70,9 +70,9 @@ Inside the DocsGPT folder create a `.env` file and copy the contents of `.env_sa
 Make sure your `.env` file looks like this:
 
 ```
-OPENAI_API_KEY=(Your OpenAI API key)
+API_KEY=<Your LLM API key>
+LLM_NAME=docsgpt
 VITE_API_STREAMING=true
-SELF_HOSTED_MODEL=false
 ```
 
 To save the file, press CTRL+X, then Y, and then ENTER.

docs/content/Deploying/DocsGPT-Settings.mdx

@@ -104,7 +104,7 @@ DocsGPT can transcribe audio in two places:
 - Voice input in the chat.
 - Audio file ingestion. Uploaded `.wav`, `.mp3`, `.m4a`, `.ogg`, and `.webm` files are transcribed first and then passed through the normal parser, chunking, embedding, and indexing pipeline.
 
-For an end-to-end walkthrough, see the [Speech and Audio Guide](/Guides/speech-and-audio).
+The settings below control speech-to-text behaviour for both voice input and audio file ingestion.
 
 | Setting | Purpose | Typical values |
 | --- | --- | --- |
@@ -214,6 +214,31 @@ If you have configured `AUTH_TYPE=simple_jwt`, the DocsGPT frontend will prompt
   }}
 />
 
+## S3 Storage Backend
+
+By default DocsGPT stores files locally. Set `STORAGE_TYPE=s3` to use Amazon S3 instead.
+
+| Setting | Description | Default |
+| --- | --- | --- |
+| `STORAGE_TYPE` | `local` or `s3` | `local` |
+| `S3_BUCKET_NAME` | S3 bucket name | `docsgpt-test-bucket` |
+| `SAGEMAKER_ACCESS_KEY` | AWS access key ID | — |
+| `SAGEMAKER_SECRET_KEY` | AWS secret access key | — |
+| `SAGEMAKER_REGION` | AWS region | — |
+| `URL_STRATEGY` | `backend` (proxy through API) or `s3` (direct S3 URLs) | `backend` |
+
+The S3 credentials use `SAGEMAKER_*` variable names because they are shared with the SageMaker integration.
+
+```env
+STORAGE_TYPE=s3
+S3_BUCKET_NAME=your-bucket-name
+SAGEMAKER_ACCESS_KEY=your-aws-access-key-id
+SAGEMAKER_SECRET_KEY=your-aws-secret-access-key
+SAGEMAKER_REGION=us-east-1
+```
+
+Your IAM user needs these permissions on the bucket: `s3:PutObject`, `s3:GetObject`, `s3:DeleteObject`, `s3:ListBucket`, `s3:HeadObject`.
+
 ## Exploring More Settings
 
 These are just the basic settings to get you started. The `settings.py` file contains many more advanced options that you can explore to further customize DocsGPT, such as:

docs/content/Deploying/Postgres-Migration.mdx

@@ -0,0 +1,114 @@
+---
+title: PostgreSQL for User Data
+description: Set up PostgreSQL as the user-data store for DocsGPT and migrate from MongoDB at your own pace.
+---
+
+import { Callout } from 'nextra/components'
+
+# PostgreSQL for User Data
+
+DocsGPT is progressively moving user data (conversations, agents, prompts,
+preferences, etc.) from MongoDB to PostgreSQL, one collection at a time.
+Each collection is guarded by a feature flag so you can opt in and roll
+back instantly. MongoDB stays the source of truth until you cut over
+reads; vector stores (`VECTOR_STORE=pgvector`, `faiss`, `qdrant`, `mongodb`, …)
+are unaffected.
+
+<Callout type="info" emoji="ℹ️">
+  Which collections are available today is in the [Status](#status)
+  table below. That table is the only part of this page that changes
+  release to release.
+</Callout>
+
+## Setup
+
+1. **Run Postgres 13+.** Native install, Docker, or managed (Neon, RDS,
+   Supabase, Cloud SQL…) — all work. You'll need the `pgcrypto` and
+   `citext` extensions, both standard contrib modules available
+   everywhere.
+
+2. **Create a database and role** (skip if your managed provider gave
+   you these):
+
+   ```sql
+   CREATE ROLE docsgpt LOGIN PASSWORD 'docsgpt';
+   CREATE DATABASE docsgpt OWNER docsgpt;
+   ```
+
+3. **Set `POSTGRES_URI` in `.env`.** Any standard Postgres URI works —
+   DocsGPT normalizes it internally.
+
+   ```bash
+   POSTGRES_URI=postgresql://docsgpt:docsgpt@localhost:5432/docsgpt
+   # Append ?sslmode=require for managed providers that enforce SSL.
+   ```
+
+4. **Apply the schema** (idempotent — safe to re-run):
+
+   ```bash
+   python scripts/db/init_postgres.py
+   ```
+
+## Migrating data
+
+Two global flags, no per-collection knobs — every collection marked ✅
+in the [Status](#status) table is handled automatically.
+
+1. **Enable dual-write.** Writes go to both Mongo and Postgres; Mongo
+   remains source of truth. Set the flag in `.env` and restart:
+
+   ```bash
+   USE_POSTGRES=true
+   ```
+
+2. **Backfill existing data.** Idempotent — re-run any time to re-sync
+   drifted rows. Without arguments, backfills every registered table;
+   pass `--tables` to limit.
+
+   ```bash
+   python scripts/db/backfill.py --dry-run           # preview everything
+   python scripts/db/backfill.py                      # real run, everything
+   python scripts/db/backfill.py --tables users      # only specific tables
+   ```
+
+3. **Cut over reads** once you trust the Postgres state:
+
+   ```bash
+   READ_POSTGRES=true
+   ```
+
+   Rollback is instant: unset `READ_POSTGRES` and restart. Dual-write
+   keeps Postgres up to date so you can flip back and forth.
+
+<Callout type="warning" emoji="⚠️">
+  Don't decommission MongoDB until every collection you use is fully
+  cut over. During the migration window, Mongo is still required.
+</Callout>
+
+## Status
+
+_Last updated: 2026-04-10_
+
+| Collection | Status |
+|---|---|
+| `users` | ✅ Phase 1 |
+| `prompts`, `user_tools`, `feedback`, `stack_logs`, `user_logs`, `token_usage` | ⏳ Phase 1 |
+| `agents`, `sources`, `attachments`, `memories`, `todos`, `notes`, `connector_sessions`, `agent_folders` | ⏳ Phase 2 |
+| `conversations`, `pending_tool_state`, `workflows` | ⏳ Phase 3 |
+
+Schemas for **every** row above already exist after `init_postgres.py`
+runs. What's landing progressively is the application-level dual-write
+wiring and the backfill logic for each collection. Once a collection
+is ✅, enabling `USE_POSTGRES=true` and running `python scripts/db/backfill.py`
+picks it up automatically — no per-collection config change.
+
+## Troubleshooting
+
+- **`relation "..." does not exist`** — run `python scripts/db/init_postgres.py`.
+- **`FATAL: role "docsgpt" does not exist`** — run the `CREATE ROLE` /
+  `CREATE DATABASE` statements from step 2 as a Postgres superuser.
+- **SSL errors on a managed provider** — append `?sslmode=require` to
+  `POSTGRES_URI`.
+- **Dual-write warnings in the logs** — expected to be non-fatal. Mongo
+  is source of truth, so the user-facing request succeeds. Re-run the
+  backfill to re-sync whichever rows drifted.

docs/content/Deploying/Railway.mdx

@@ -86,13 +86,9 @@ Make sure your `.env` file looks like this:
   
 
 ```
-
-OPENAI_API_KEY=(Your OpenAI API key)
-
+API_KEY=<Your LLM API key>
+LLM_NAME=docsgpt
 VITE_API_STREAMING=true
-
-SELF_HOSTED_MODEL=false
-
 ```
 
   

docs/content/Deploying/_meta.js

@@ -19,6 +19,10 @@ export default {
     "title": "☁️ Hosting DocsGPT",
     "href": "/Deploying/Hosting-the-app"
   },
+  "Postgres-Migration": {
+    "title": "🐘 PostgreSQL for User Data",
+    "href": "/Deploying/Postgres-Migration"
+  },
   "Amazon-Lightsail": {
     "title": "Hosting DocsGPT on Amazon Lightsail",
     "href": "/Deploying/Amazon-Lightsail",

docs/content/Extensions/api-key-guide.mdx

@@ -11,18 +11,18 @@ DocsGPT API keys are essential for developers and users who wish to integrate th
 
 After uploading your document, you can obtain an API key either through the graphical user interface or via an API call:
 
-- **Graphical User Interface:** Navigate to the Settings section of the DocsGPT web app, find the API Keys option, and press 'Create New' to generate your key.
-- **API Call:** Alternatively, you can use the `/api/create_api_key` endpoint to create a new API key. For detailed instructions, visit [DocsGPT API Documentation](https://gptcloud.arc53.com/).
+- **Graphical User Interface:** Navigate to the Settings section of the DocsGPT web app, find the Agents option, and press 'Create New' to generate a new agent (which includes an API key).
+- **API Call:** Alternatively, you can use the `/api/create_agent` endpoint to create a new agent. An API key is automatically generated for each agent. For detailed instructions, visit [DocsGPT API Documentation](https://gptcloud.arc53.com/).
 
 ## Understanding Key Variables
 
-Upon creating your API key, you will encounter several key variables. Each serves a specific purpose:
+Upon creating your agent, you will encounter several key variables. Each serves a specific purpose:
 
-- **Name:** Assign a name to your API key for easy identification.
-- **Source:** Indicates the source document(s) linked to your API key, which DocsGPT will use to generate responses.
-- **ID:** A unique identifier for your API key. You can view this by making a call to `/api/get_api_keys`.
-- **Key:** The API key itself, which will be used in your application to authenticate API requests.
+- **Name:** Assign a name to your agent for easy identification.
+- **Source:** Indicates the source document(s) linked to your agent, which DocsGPT will use to generate responses.
+- **ID:** A unique identifier for your agent. You can view this by making a call to `/api/get_agents`.
+- **Key:** The API key for the agent, which will be used in your application to authenticate API requests.
 
-With your API key ready, you can now integrate DocsGPT into your application, such as the DocsGPT Widget or any other software, via `/api/answer` or `/stream` endpoints. The source document is preset with the API key, allowing you to bypass fields like `selectDocs` and `active_docs` during implementation.
+With your API key ready, you can now integrate DocsGPT into your application, such as the DocsGPT Widget or any other software, via `/api/answer` or `/stream` endpoints. The source document is preset with the agent, allowing you to bypass fields like `selectDocs` and `active_docs` during implementation.
 
 Congratulations on taking the first step towards enhancing your applications with DocsGPT!

docs/content/Guides/Architecture.mdx

@@ -64,7 +64,7 @@ flowchart LR
 *   **Technology:** Supports multiple vector databases.
 *   **Responsibility:** Vector Stores are used to store and retrieve vector embeddings of document chunks. This enables semantic search and retrieval of relevant document snippets in response to user queries.
 *   **Key Features:**
-    *   Supports vector databases including FAISS, Elasticsearch, Qdrant, Milvus, and LanceDB.
+    *   Supports vector databases including FAISS, Elasticsearch, Qdrant, Milvus, MongoDB Atlas Vector Search, and pgvector.
     *   Provides storage and indexing of high-dimensional vector embeddings.
     *   Enables editing and updating of vector indexes including specific chunks.
 

docs/content/Guides/How-to-train-on-other-documentation.mdx

@@ -16,7 +16,7 @@ Training on other documentation sources can greatly enhance the versatility and
 Make sure you have the document on which you want to train on  ready with you on the device which you are using .You can also use links to the documentation to train on.
 
 <Callout type="warning" emoji="⚠️">
- Note: The document should be either of the given file formats .pdf, .txt, .rst, .docx, .md, .zip and  limited to 25mb.You can also train using the link of the documentation.
+ Note: Supported file formats include .pdf, .txt, .rst, .docx, .md, .mdx, .csv, .epub, .html, .json, .xlsx, .pptx, .png, .jpg, .jpeg, and audio files (.wav, .mp3, .m4a, .ogg, .webm). You can also train using the link of the documentation.
 
 </Callout>
 

docs/content/Guides/How-to-use-different-LLM.mdx

@@ -35,8 +35,34 @@ Choose the LLM of your choice.
 For open source version please edit `LLM_PROVIDER`, `LLM_NAME` and others in the .env file. Refer to [⚙️ App Configuration](/Deploying/DocsGPT-Settings) for more information.
 ### Step 2
 Visit [☁️ Cloud Providers](/Models/cloud-providers) for the updated list of online models. Make sure you have the right API_KEY and correct LLM_PROVIDER.
-For self-hosted please visit [🖥️ Local Inference](/Models/local-inference). 
+For self-hosted please visit [🖥️ Local Inference](/Models/local-inference).
 </Steps>
 
+## Fallback LLM
+
+DocsGPT can automatically switch to a fallback LLM when the primary model fails, including mid-stream. This works with both streaming and non-streaming requests.
+
+**Fallback order:**
+1. Per-agent backup models (other models configured on the same agent)
+2. Global fallback (`FALLBACK_LLM_*` env vars below)
+3. Error returned if all fail
+
+| Setting | Description | Default |
+| --- | --- | --- |
+| `FALLBACK_LLM_PROVIDER` | Provider name (e.g., `openai`, `anthropic`, `google`) | — |
+| `FALLBACK_LLM_NAME` | Model name (e.g., `gpt-4o`, `claude-sonnet-4-20250514`) | — |
+| `FALLBACK_LLM_API_KEY` | API key for the fallback provider | Falls back to `API_KEY` |
+
+All three (`FALLBACK_LLM_PROVIDER`, `FALLBACK_LLM_NAME`, and an API key) must resolve for the global fallback to activate.
+
+```env
+FALLBACK_LLM_PROVIDER=anthropic
+FALLBACK_LLM_NAME=claude-sonnet-4-20250514
+FALLBACK_LLM_API_KEY=sk-ant-your-anthropic-key
+```
+
+<Callout type="info">
+For maximum resilience, use a fallback provider from a different cloud than your primary. Each agent can also have multiple models configured — the other models are tried first before the global fallback.
+</Callout>
 
 

docs/content/Guides/Integrations/_meta.js

@@ -2,5 +2,13 @@ export default {
   "google-drive-connector": {
     "title": "🔗 Google Drive",
     "href": "/Guides/Integrations/google-drive-connector"
+  },
+  "sharepoint-connector": {
+    "title": "🔗 SharePoint / OneDrive",
+    "href": "/Guides/Integrations/sharepoint-connector"
+  },
+  "mcp-tool-integration": {
+    "title": "🔗 MCP Tools",
+    "href": "/Guides/Integrations/mcp-tool-integration"
   }
 }

docs/content/Guides/Integrations/mcp-tool-integration.mdx

@@ -0,0 +1,66 @@
+---
+title: MCP Tool Integration
+description: Connect external tools to DocsGPT agents using the Model Context Protocol (MCP) standard.
+---
+
+import { Callout } from 'nextra/components'
+import { Steps } from 'nextra/components'
+
+# MCP Tool Integration
+
+The [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) integration lets you connect external tool servers to DocsGPT. Your agents can then discover and call tools provided by those servers during conversations — for example, querying a CRM, running code, or accessing a database.
+
+## Setup
+
+<Steps>
+
+### Step 1: Configure Environment Variables (Optional)
+
+Only needed if your MCP servers use OAuth authentication:
+
+```env
+MCP_OAUTH_REDIRECT_URI=https://yourdomain.com/api/mcp_server/callback
+```
+
+If not set, falls back to `API_URL/api/mcp_server/callback`.
+
+### Step 2: Add an MCP Server
+
+Go to **Settings** > **Tools** > **Add Tool** > **MCP Server**. Enter the server URL, select an auth type, and click **Test Connection** to verify, then **Save**.
+
+### Step 3: Enable for Your Agent
+
+In your agent configuration, enable the MCP tools you want the agent to use.
+
+</Steps>
+
+## Authentication Types
+
+| Auth Type | Config Fields |
+|-----------|---------------|
+| **None** | — |
+| **Bearer** | `bearer_token` |
+| **API Key** | `api_key`, `api_key_header` (default: `X-API-Key`) |
+| **Basic** | `username`, `password` |
+| **OAuth** | `oauth_scopes` (optional) |
+
+<Callout type="warning">
+For OAuth in production, `MCP_OAUTH_REDIRECT_URI` must be a publicly accessible URL pointing to your DocsGPT backend.
+</Callout>
+
+## API Endpoints
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/api/mcp_server/test` | POST | Test a connection without saving |
+| `/api/mcp_server/save` | POST | Save or update a server configuration |
+| `/api/mcp_server/callback` | GET | OAuth callback handler |
+| `/api/mcp_server/oauth_status/<task_id>` | GET | Poll OAuth flow status |
+| `/api/mcp_server/auth_status` | GET | Batch check auth status for all MCP tools |
+
+## Troubleshooting
+
+- **Connection refused** — Verify the URL and that the server is reachable from your backend.
+- **403 Forbidden** — Check credentials and permissions.
+- **Timed out** — Default is 30s; increase timeout in tool config (max 300s).
+- **OAuth "needs_auth" persists** — Verify `MCP_OAUTH_REDIRECT_URI` is correct and Redis is running.

docs/content/Guides/Integrations/sharepoint-connector.mdx

@@ -0,0 +1,63 @@
+---
+title: SharePoint / OneDrive Connector
+description: Connect your Microsoft SharePoint or OneDrive as an external knowledge base to upload and process files directly.
+---
+
+import { Callout } from 'nextra/components'
+import { Steps } from 'nextra/components'
+
+# SharePoint / OneDrive Connector
+
+Connect your SharePoint or OneDrive account to upload and process files directly as an external knowledge base. Supports Office files, PDFs, text files, CSVs, images, and more. Authentication is handled via Microsoft Entra ID (Azure AD) with automatic token refresh.
+
+## Setup
+
+<Steps>
+
+### Step 1: Create an App Registration in Azure
+
+1. Go to the [Azure Portal](https://portal.azure.com/) > **Microsoft Entra ID** > **App registrations** > **New registration**
+2. Set **Redirect URI** (Web) to:
+   - Local: `http://localhost:7091/api/connectors/callback?provider=share_point`
+   - Production: `https://yourdomain.com/api/connectors/callback?provider=share_point`
+
+### Step 2: Configure API Permissions
+
+In your App Registration, go to **API permissions** > **Add a permission** > **Microsoft Graph** > **Delegated permissions** and add: `Files.Read`, `Files.Read.All`, `Sites.Read.All`. Grant admin consent if possible.
+
+### Step 3: Create a Client Secret
+
+Go to **Certificates & secrets** > **New client secret**. Copy the secret value immediately (it won't be shown again).
+
+### Step 4: Configure Environment Variables
+
+Add to your `.env` file:
+
+```env
+MICROSOFT_CLIENT_ID=your-azure-ad-client-id
+MICROSOFT_CLIENT_SECRET=your-azure-ad-client-secret
+MICROSOFT_TENANT_ID=your-azure-ad-tenant-id
+```
+
+| Variable | Description | Required | Default |
+|----------|-------------|----------|---------|
+| `MICROSOFT_CLIENT_ID` | Application (client) ID from App Registration overview | Yes | — |
+| `MICROSOFT_CLIENT_SECRET` | Client secret value | Yes | — |
+| `MICROSOFT_TENANT_ID` | Directory (tenant) ID | No | `common` |
+| `MICROSOFT_AUTHORITY` | Login endpoint override | No | Auto-constructed |
+
+<Callout type="warning">
+`MICROSOFT_TENANT_ID=common` (the default) allows any Microsoft account to authenticate. Set this to your specific tenant ID in production.
+</Callout>
+
+### Step 5: Restart and Use
+
+Restart your application, then go to the upload section in DocsGPT and select **SharePoint / OneDrive** as the source. You'll be redirected to Microsoft to sign in, then can browse and select files to process.
+
+</Steps>
+
+## Troubleshooting
+
+- **Option not appearing** — Verify `MICROSOFT_CLIENT_ID` and `MICROSOFT_CLIENT_SECRET` are set, then restart.
+- **Authentication failed** — Check that the redirect URI matches exactly, including `?provider=share_point`.
+- **Permission denied** — Ensure admin consent is granted and the user has access to the target files.

docs/content/Guides/My-AI-answers-questions-using-external-knowledge.mdx

@@ -7,20 +7,10 @@ description:
 
 If your AI uses external knowledge and is not explicit enough, it is ok, because we try to make DocsGPT friendly.
 
-But if you want to adjust it, here is a simple way:-
-
-- Got to `application/prompts/chat_combine_prompt.txt`
-
-- And change it to
+But if you want to adjust it, prompts are now managed through the UI and API using a template-based system. See the [Customising Prompts](/Guides/Customising-prompts) guide for details.
 
+To make the AI stricter about staying on-topic, edit your active prompt template (via **Sidebar → Settings → Active Prompt**) to include instructions like:
 
 ```
-
-You are a DocsGPT, friendly and helpful AI assistant by Arc53 that provides help with documents. You give thorough answers with code examples, if possible.
-Write an answer for the question below based on the provided context.
 If the context provides insufficient information, reply "I cannot answer".
-You have access to chat history and can use it to help answer the question.
-----------------
-{summaries}
-
 ```

docs/content/Guides/_meta.js

@@ -29,7 +29,7 @@ export default {
     "title": "OCR",
     "href": "/Guides/ocr"
   },
-  "Integrations": {
+"Integrations": {
     "title": "🔗 Integrations"
   }
 }

docs/content/quickstart.mdx

@@ -70,7 +70,7 @@ The easiest way to launch DocsGPT is using the provided `setup.sh` script. This
    To stop DocsGPT, simply open a new terminal in the `DocsGPT` directory and run:
 
    ```bash
-   docker compose -f deployment/docker-compose.yaml down
+   docker compose -f deployment/docker-compose-hub.yaml down
    ```
    (or the specific `docker compose` command shown at the end of the `setup.sh` execution, which may include optional compose files depending on your choices).
 

extensions/chatwoot/app.py

@@ -1,3 +1,5 @@
+import hashlib
+import hmac
 import os
 import pprint
 
@@ -10,6 +12,7 @@ docsgpt_url = os.getenv("docsgpt_url")
 chatwoot_url = os.getenv("chatwoot_url")
 docsgpt_key = os.getenv("docsgpt_key")
 chatwoot_token = os.getenv("chatwoot_token")
+chatwoot_webhook_secret = os.getenv("chatwoot_webhook_secret", "")
 # account_id = os.getenv("account_id")
 # assignee_id = os.getenv("assignee_id")
 label_stop = "human-requested"
@@ -45,12 +48,35 @@ def send_to_chatwoot(account, conversation, message):
     return r.json()
 
 
+def is_valid_chatwoot_signature(raw_body: bytes, signature_header: str | None) -> bool:
+    """Validate Chatwoot webhook signature using shared secret."""
+    if not chatwoot_webhook_secret or not signature_header:
+        return False
+
+    expected = hmac.new(
+        chatwoot_webhook_secret.encode("utf-8"), raw_body, hashlib.sha256
+    ).hexdigest()
+
+    provided = signature_header.strip()
+    if provided.startswith("sha256="):
+        provided = provided.split("=", maxsplit=1)[1]
+
+    return hmac.compare_digest(provided, expected)
+
+
 app = Flask(__name__)
 
 
 @app.route('/docsgpt', methods=['POST'])
 def docsgpt():
-    data = request.get_json()
+    raw_body = request.get_data()
+    signature = request.headers.get("X-Chatwoot-Signature")
+    if not is_valid_chatwoot_signature(raw_body, signature):
+        return "Unauthorized", 401
+
+    data = request.get_json(silent=True)
+    if not isinstance(data, dict):
+        return "Invalid payload", 400
     pp = pprint.PrettyPrinter(indent=4)
     pp.pprint(data)
     try:

extensions/react-widget/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "docsgpt",
-  "version": "0.5.1",
+  "version": "0.6.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "docsgpt",
-      "version": "0.5.1",
+      "version": "0.6.3",
       "license": "Apache-2.0",
       "dependencies": {
         "@babel/plugin-transform-flow-strip-types": "^7.23.3",

extensions/react-widget/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docsgpt",
-  "version": "0.6.1",
+  "version": "0.6.3",
   "private": false,
   "description": "DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.",
   "source": "./src/index.html",

extensions/slack-bot/requirements.txt

@@ -1,6 +1,6 @@
 aiohttp>=3,<4
 certifi==2024.7.4
-h11==0.16.0
+h11==0.14.0
 httpcore==1.0.5
 httpx==0.27.0
 idna==3.7

frontend/src/App.tsx

@@ -37,7 +37,7 @@ function MainLayout() {
   const [navOpen, setNavOpen] = useState(!(isMobile || isTablet));
 
   return (
-    <div className="dark:bg-raisin-black relative h-screen overflow-hidden">
+    <div className="bg-background relative h-screen overflow-hidden">
       <Navigation navOpen={navOpen} setNavOpen={setNavOpen} />
       <ActionButtons showNewChat={true} showShare={true} />
       <div

frontend/src/Hero.tsx

@@ -21,10 +21,10 @@ export default function Hero({
   }>;
 
   return (
-    <div className="text-black-1000 dark:text-bright-gray flex h-full w-full flex-col items-center justify-between">
+    <div className="text-black-1000 dark:text-foreground flex h-full w-full flex-col items-center justify-between">
       {/* Header Section */}
       <div className="flex grow flex-col items-center justify-center pt-8 md:pt-0">
-        <div className="mb-4 flex items-center">
+        <div className="mb-px flex items-center">
           <span className="text-4xl font-semibold">DocsGPT</span>
           <img className="mb-1 inline w-14" src={DocsGPT3} alt="docsgpt" />
         </div>
@@ -44,9 +44,9 @@ export default function Hero({
                 <button
                   key={key}
                   onClick={() => handleQuestion({ question: demo.query })}
-                  className={`border-dark-gray text-just-black hover:bg-cultured dark:border-dim-gray dark:text-chinese-white dark:hover:bg-charleston-green w-full rounded-[66px] border bg-transparent px-6 py-[14px] text-left transition-colors ${key >= 2 ? 'hidden md:block' : ''}`}
+                  className={`border-border text-foreground hover:bg-muted dark:hover:bg-muted/50 bg-card w-full rounded-[66px] border px-6 py-3.5 text-left transition-colors dark:bg-transparent ${key >= 2 ? 'hidden md:block' : ''}`}
                 >
-                  <p className="text-black-1000 dark:text-bright-gray mb-2 font-semibold">
+                  <p className="text-black-1000 dark:text-foreground mb-2 font-semibold">
                     {demo.header}
                   </p>
                   <span className="line-clamp-2 text-gray-700 opacity-60 dark:text-gray-300">

frontend/src/Navigation.tsx

@@ -328,7 +328,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                 />
               </button>
             )}
-            <div className="text-gray-4000 text-[20px] font-medium">
+            <div className="text-muted-foreground text-[20px] font-medium">
               DocsGPT
             </div>
           </div>
@@ -338,7 +338,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         ref={navRef}
         className={`${
           !navOpen && '-ml-96 md:-ml-72'
-        } bg-lotion dark:border-r-purple-taupe dark:bg-chinese-black fixed top-0 z-20 flex h-full w-72 flex-col border-r border-b-0 transition-all duration-300 ease-in-out dark:text-white`}
+        } bg-sidebar dark:border-r-sidebar-border fixed top-0 z-20 flex h-full w-72 flex-col border-r border-b-0 transition-all duration-300 ease-in-out dark:text-white`}
       >
         <div
           className={'visible mt-2 flex h-[6vh] w-full justify-between md:h-12'}
@@ -380,7 +380,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
           className={({ isActive }) =>
             `${
               isActive ? 'bg-transparent' : ''
-            } group border-silver hover:border-rainy-gray dark:border-purple-taupe sticky mx-4 mt-4 flex cursor-pointer gap-2.5 rounded-3xl border p-3 hover:bg-transparent dark:text-white`
+            } group border-sidebar-border hover:border-sidebar-border sticky mx-4 mt-4 flex cursor-pointer gap-2.5 rounded-3xl border p-3 hover:bg-transparent dark:text-white`
           }
         >
           <img
@@ -388,13 +388,13 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             alt="Create new chat"
             className="opacity-80 group-hover:opacity-100"
           />
-          <p className="text-dove-gray dark:text-chinese-silver dark:group-hover:text-bright-gray text-sm group-hover:text-neutral-600">
+          <p className="text-muted-foreground dark:text-foreground dark:group-hover:text-foreground text-sm group-hover:text-neutral-600">
             {t('newChat')}
           </p>
         </NavLink>
         <div
           id="conversationsMainDiv"
-          className="mb-auto h-[78vh] overflow-x-hidden overflow-y-auto scrollbar-overlay dark:text-white"
+          className="scrollbar-overlay mb-auto h-[78vh] overflow-x-hidden overflow-y-auto dark:text-white"
         >
           {conversations?.loading && !isDeletingConversation && (
             <div className="absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2 transform">
@@ -417,9 +417,9 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                   {recentAgents.map((agent, idx) => (
                     <div
                       key={idx}
-                      className={`group hover:bg-bright-gray dark:hover:bg-dark-charcoal mx-4 my-auto mt-4 flex h-9 cursor-pointer items-center justify-between rounded-3xl pl-4 ${
+                      className={`group hover:bg-sidebar-accent mx-4 my-auto mt-4 flex h-9 cursor-pointer items-center justify-between rounded-3xl pl-4 ${
                         agent.id === selectedAgent?.id && !conversationId
-                          ? 'bg-bright-gray dark:bg-dark-charcoal'
+                          ? 'bg-sidebar-accent'
                           : ''
                       }`}
                       onClick={() => handleAgentClick(agent)}
@@ -432,7 +432,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                             className="h-6 w-6 rounded-full object-contain"
                           />
                         </div>
-                        <p className="text-eerie-black dark:text-bright-gray overflow-hidden text-sm leading-6 text-ellipsis whitespace-nowrap">
+                        <p className="text-foreground dark:text-foreground overflow-hidden text-sm leading-6 text-ellipsis whitespace-nowrap">
                           {agent.name}
                         </p>
                       </div>
@@ -456,7 +456,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                   ))}
                 </div>
                 <div
-                  className="hover:bg-bright-gray dark:hover:bg-dark-charcoal mx-4 my-auto mt-2 flex h-9 cursor-pointer items-center gap-2 rounded-3xl pl-4"
+                  className="hover:bg-sidebar-accent mx-4 my-auto mt-2 flex h-9 cursor-pointer items-center gap-2 rounded-3xl pl-4"
                   onClick={() => {
                     dispatch(setSelectedAgent(null));
                     if (isMobile || isTablet) {
@@ -472,7 +472,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                       className="h-[18px] w-[18px]"
                     />
                   </div>
-                  <p className="text-eerie-black dark:text-bright-gray overflow-hidden text-sm leading-6 text-ellipsis whitespace-nowrap">
+                  <p className="text-foreground dark:text-foreground overflow-hidden text-sm leading-6 text-ellipsis whitespace-nowrap">
                     {t('manageAgents')}
                   </p>
                 </div>
@@ -480,7 +480,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             </div>
           ) : (
             <div
-              className="hover:bg-bright-gray dark:hover:bg-dark-charcoal mx-4 my-auto mt-2 flex h-9 cursor-pointer items-center gap-2 rounded-3xl pl-4"
+              className="hover:bg-sidebar-accent mx-4 my-auto mt-2 flex h-9 cursor-pointer items-center gap-2 rounded-3xl pl-4"
               onClick={() => {
                 if (isMobile || isTablet) {
                   setNavOpen(false);
@@ -496,7 +496,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                   className="h-[18px] w-[18px]"
                 />
               </div>
-              <p className="text-eerie-black dark:text-bright-gray overflow-hidden text-sm leading-6 text-ellipsis whitespace-nowrap">
+              <p className="text-foreground dark:text-foreground overflow-hidden text-sm leading-6 text-ellipsis whitespace-nowrap">
                 {t('manageAgents')}
               </p>
             </div>
@@ -529,8 +529,8 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
             <></>
           )}
         </div>
-        <div className="text-eerie-black flex h-auto flex-col justify-end dark:text-white">
-          <div className="dark:border-b-purple-taupe flex flex-col gap-2 border-b py-2">
+        <div className="text-foreground flex h-auto flex-col justify-end dark:text-white">
+          <div className="dark:border-b-sidebar-border flex flex-col gap-2 border-b py-2">
             <NavLink
               onClick={() => {
                 if (isMobile || isTablet) {
@@ -540,8 +540,8 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
               }}
               to="/settings"
               className={({ isActive }) =>
-                `mx-4 my-auto flex h-9 cursor-pointer items-center gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
-                  isActive ? 'bg-gray-3000 dark:bg-transparent' : ''
+                `hover:bg-sidebar-accent mx-4 my-auto flex h-9 cursor-pointer items-center gap-4 rounded-3xl ${
+                  isActive ? 'bg-sidebar-accent' : ''
                 }`
               }
             >
@@ -552,12 +552,12 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                 height={21}
                 className="my-auto ml-2 filter dark:invert"
               />
-              <p className="text-eerie-black text-sm dark:text-white">
+              <p className="text-foreground text-sm dark:text-white">
                 {t('settings.label')}
               </p>
             </NavLink>
           </div>
-          <div className="text-eerie-black flex flex-col justify-end dark:text-white">
+          <div className="text-foreground flex flex-col justify-end dark:text-white">
             <div className="flex items-center justify-between py-1">
               <Help />
 
@@ -565,9 +565,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                 <NavLink
                   target="_blank"
                   to={'https://discord.gg/vN7YFfdMpj'}
-                  className={
-                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                  }
+                  className={'hover:bg-sidebar-accent rounded-full'}
                 >
                   <img
                     src={Discord}
@@ -580,9 +578,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                 <NavLink
                   target="_blank"
                   to={'https://x.com/docsgptai'}
-                  className={
-                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                  }
+                  className={'hover:bg-sidebar-accent rounded-full'}
                 >
                   <img
                     src={Twitter}
@@ -595,9 +591,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                 <NavLink
                   target="_blank"
                   to={'https://github.com/arc53/docsgpt'}
-                  className={
-                    'rounded-full hover:bg-gray-100 dark:hover:bg-[#28292E]'
-                  }
+                  className={'hover:bg-sidebar-accent rounded-full'}
                 >
                   <img
                     src={Github}
@@ -612,7 +606,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
           </div>
         </div>
       </div>
-      <div className="dark:border-b-purple-taupe dark:bg-chinese-black sticky z-10 h-16 w-full border-b-2 bg-gray-50 lg:hidden">
+      <div className="dark:border-b-sidebar-border bg-sidebar sticky z-10 h-16 w-full border-b-2 lg:hidden">
         <div className="ml-6 flex h-full items-center gap-6">
           <button
             className="h-6 w-6 lg:hidden"
@@ -624,7 +618,9 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
               className="w-7 filter dark:invert"
             />
           </button>
-          <div className="text-gray-4000 text-[20px] font-medium">DocsGPT</div>
+          <div className="text-muted-foreground text-[20px] font-medium">
+            DocsGPT
+          </div>
         </div>
       </div>
       <DeleteConvModal

frontend/src/PageNotFound.tsx

@@ -5,8 +5,8 @@ export default function PageNotFound() {
   const { t } = useTranslation();
 
   return (
-    <div className="dark:bg-raisin-black grid min-h-screen">
-      <p className="text-jet dark:bg-outer-space mx-auto my-auto mt-20 flex w-full max-w-6xl flex-col place-items-center gap-6 rounded-3xl bg-gray-100 p-6 lg:p-10 xl:p-16 dark:text-gray-100">
+    <div className="bg-background grid min-h-screen">
+      <p className="text-foreground dark:bg-card mx-auto my-auto mt-20 flex w-full max-w-6xl flex-col place-items-center gap-6 rounded-3xl bg-gray-100 p-6 lg:p-10 xl:p-16">
         <h1>{t('pageNotFound.title')}</h1>
         <p>{t('pageNotFound.message')}</p>
         <button className="pointer-cursor bg-blue-1000 hover:bg-blue-3000 mr-4 flex cursor-pointer items-center justify-center rounded-full px-4 py-2 text-white transition-colors duration-100">

frontend/src/agents/AgentCard.tsx

@@ -251,7 +251,7 @@ export default function AgentCard({
   };
   return (
     <div
-      className={`relative flex h-44 flex-col justify-between rounded-[1.2rem] bg-[#F6F6F6] px-4 py-5 hover:bg-[#ECECEC] sm:w-48 sm:px-6 dark:bg-[#383838] dark:hover:bg-[#383838]/80 ${agent.status === 'published' && 'cursor-pointer'}`}
+      className={`bg-muted hover:bg-accent relative flex h-44 flex-col justify-between rounded-[1.2rem] px-4 py-5 sm:w-48 sm:px-6 ${agent.status === 'published' && 'cursor-pointer'}`}
       onClick={(e) => {
         e.stopPropagation();
         handleClick();
@@ -283,17 +283,17 @@ export default function AgentCard({
             className="h-7 w-7 rounded-full object-contain"
           />
           {agent.status === 'draft' && (
-            <p className="text-xs text-black opacity-50 dark:text-[#E0E0E0]">{`(Draft)`}</p>
+            <p className="text-foreground text-xs opacity-50">{`(Draft)`}</p>
           )}
         </div>
         <div className="mt-2">
           <p
             title={agent.name}
-            className="truncate px-1 text-[13px] leading-relaxed font-semibold text-[#020617] capitalize dark:text-[#E0E0E0]"
+            className="text-foreground truncate px-1 text-[13px] leading-relaxed font-semibold capitalize"
           >
             {agent.name}
           </p>
-          <p className="dark:text-sonic-silver-light mt-1 h-20 overflow-auto px-1 text-[12px] leading-relaxed text-[#64748B]">
+          <p className="dark:text-muted-foreground text-muted-foreground mt-1 h-20 overflow-auto px-1 text-[12px] leading-relaxed">
             {agent.description}
           </p>
         </div>
@@ -320,4 +320,4 @@ export default function AgentCard({
       />
     </div>
   );
-}
+}

frontend/src/agents/AgentLogs.tsx

@@ -41,25 +41,25 @@ export default function AgentLogs() {
     <div className="p-4 md:p-12">
       <div className="flex items-center gap-3 px-4">
         <button
-          className="rounded-full border p-3 text-sm text-gray-400 dark:border-0 dark:bg-[#28292D] dark:text-gray-500 dark:hover:bg-[#2E2F34]"
+          className="border-border text-muted-foreground hover:bg-accent rounded-full border p-3 text-sm"
           onClick={() => navigate('/agents')}
         >
           <img src={ArrowLeft} alt="left-arrow" className="h-3 w-3" />
         </button>
-        <p className="text-eerie-black dark:text-bright-gray mt-px text-sm font-semibold">
+        <p className="text-foreground dark:text-foreground mt-px text-sm font-semibold">
           {t('agents.backToAll')}
         </p>
       </div>
       <div className="mt-5 flex w-full flex-wrap items-center justify-between gap-2 px-4">
-        <h1 className="text-eerie-black m-0 text-[32px] font-bold md:text-[40px] dark:text-white">
+        <h1 className="text-foreground m-0 text-[32px] font-bold md:text-[40px] dark:text-white">
           {t('agents.logs.title')}
         </h1>
       </div>
       <div className="mt-6 flex flex-col gap-3 px-4">
         {agent && (
           <div className="flex flex-col gap-1">
-            <p className="text-[#28292E] dark:text-[#E0E0E0]">{agent.name}</p>
-            <p className="text-xs text-[#28292E] dark:text-[#E0E0E0]/40">
+            <p className="text-foreground">{agent.name}</p>
+            <p className="text-muted-foreground text-xs">
               {agent.last_used_at
                 ? t('agents.logs.lastUsedAt') +
                   ' ' +

frontend/src/agents/AgentPreview.tsx

@@ -131,7 +131,7 @@ export default function AgentPreview() {
             autoFocus={false}
           />
         </div>
-        <p className="text-gray-4000 dark:text-sonic-silver w-full bg-transparent text-center text-xs md:inline">
+        <p className="text-muted-foreground w-full bg-transparent text-center text-xs md:inline">
           {t('agents.preview.testMessage')}
         </p>
       </div>

frontend/src/agents/AgentsList.tsx

@@ -159,10 +159,10 @@ export default function AgentsList() {
 
   return (
     <div className="p-4 md:p-12">
-      <h1 className="text-eerie-black mb-0 text-[32px] font-bold lg:text-[40px] dark:text-[#E0E0E0]">
+      <h1 className="text-foreground mb-0 text-[32px] font-bold lg:text-[40px]">
         {t('agents.title')}
       </h1>
-      <p className="dark:text-gray-4000 mt-5 text-[15px] leading-6 text-[#71717A]">
+      <p className="text-muted-foreground mt-5 text-[15px] leading-6">
         {t('agents.description')}
       </p>
 
@@ -178,7 +178,7 @@ export default function AgentsList() {
             value={searchQuery}
             onChange={(e) => setSearchQuery(e.target.value)}
             placeholder={t('agents.searchPlaceholder')}
-            className="h-11 w-full rounded-full border border-[#E5E5E5] bg-white py-2 pr-5 pl-11 text-sm shadow-[0_1px_4px_rgba(0,0,0,0.06)] transition-shadow outline-none placeholder:text-[#9CA3AF] focus:shadow-[0_2px_8px_rgba(0,0,0,0.1)] dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white dark:shadow-none dark:placeholder:text-[#6B7280]"
+            className="border-border bg-card text-foreground placeholder:text-muted-foreground h-11 w-full rounded-full border py-2 pr-5 pl-11 text-sm shadow-[0_1px_4px_rgba(0,0,0,0.06)] transition-shadow outline-none focus:shadow-[0_2px_8px_rgba(0,0,0,0.1)] dark:shadow-none"
           />
         </div>
 
@@ -189,8 +189,8 @@ export default function AgentsList() {
               onClick={() => setActiveFilter(tab.id)}
               className={`rounded-full px-4 py-2 text-sm transition-colors ${
                 activeFilter === tab.id
-                  ? 'bg-[#E0E0E0] text-[#18181B] dark:bg-[#4A4A4A] dark:text-white'
-                  : 'dark:text-gray bg-transparent text-[#71717A] hover:bg-[#F5F5F5] dark:hover:bg-[#383838]/50'
+                  ? 'bg-border text-foreground dark:bg-accent dark:text-white'
+                  : 'dark:text-gray text-muted-foreground hover:bg-accent/50 bg-transparent'
               }`}
             >
               {t(tab.labelKey)}
@@ -224,7 +224,7 @@ export default function AgentsList() {
       ))}
 
       {showSearchEmptyState && (
-        <div className="mt-12 flex flex-col items-center justify-center gap-2 text-[#71717A]">
+        <div className="text-muted-foreground mt-12 flex flex-col items-center justify-center gap-2">
           <p className="text-lg">{t('agents.noSearchResults')}</p>
           <p className="text-sm">{t('agents.tryDifferentSearch')}</p>
         </div>
@@ -399,7 +399,7 @@ function AgentSection({
 
   if (isFilteredView && isSearchingWithNoResults) {
     return (
-      <div className="mt-12 flex flex-col items-center justify-center gap-2 text-[#71717A]">
+      <div className="text-muted-foreground mt-12 flex flex-col items-center justify-center gap-2">
         <p className="text-lg">{t('agents.noSearchResults')}</p>
         <p className="text-sm">{t('agents.tryDifferentSearch')}</p>
       </div>
@@ -408,11 +408,11 @@ function AgentSection({
 
   if (isFilteredView && hasNoAgentsAtAll) {
     return (
-      <div className="mt-12 flex flex-col items-center justify-center gap-3 text-[#71717A]">
+      <div className="text-muted-foreground mt-12 flex flex-col items-center justify-center gap-3">
         <p>{t(`agents.sections.${config.id}.emptyState`)}</p>
         {config.showNewAgentButton && (
           <button
-            className="bg-purple-30 hover:bg-violets-are-blue rounded-full px-4 py-2 text-sm text-white"
+            className="bg-primary hover:bg-primary/90 rounded-full px-4 py-2 text-sm text-white"
             onClick={() => {
               setModalFolderId(null);
               setShowAgentTypeModal(true);
@@ -456,12 +456,12 @@ function AgentSection({
     <div className="mt-8 flex flex-col gap-4">
       <div className="flex w-full flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
         <div className="flex flex-col gap-2">
-          <h2 className="flex flex-wrap items-center gap-2 text-[18px] font-semibold text-[#18181B] dark:text-[#E0E0E0]">
+          <h2 className="text-foreground flex flex-wrap items-center gap-2 text-[18px] font-semibold">
             {config.id === 'user' && folderPath.length > 0 ? (
               <>
                 <button
                   onClick={() => handleNavigateToPath(-1)}
-                  className="text-[#71717A] hover:text-[#18181B] dark:hover:text-white"
+                  className="text-muted-foreground hover:text-foreground dark:hover:text-white"
                 >
                   {t(`agents.sections.${config.id}.title`)}
                 </button>
@@ -473,7 +473,7 @@ function AgentSection({
                     ) : (
                       <button
                         onClick={() => handleNavigateToPath(index)}
-                        className="text-[#71717A] hover:text-[#18181B] dark:hover:text-white"
+                        className="text-muted-foreground hover:text-foreground dark:hover:text-white"
                       >
                         {item.name}
                       </button>
@@ -485,7 +485,7 @@ function AgentSection({
               t(`agents.sections.${config.id}.title`)
             )}
           </h2>
-          <p className="text-[13px] text-[#71717A]">
+          <p className="text-muted-foreground text-[13px]">
             {t(`agents.sections.${config.id}.description`)}
           </p>
         </div>
@@ -513,12 +513,12 @@ function AgentSection({
                   }
                 }}
                 placeholder={t('agents.folders.newFolder')}
-                className="w-28 rounded-full border border-[#E5E5E5] bg-white px-4 py-2 text-sm text-[#18181B] outline-none placeholder:text-[#9CA3AF] sm:w-auto dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white dark:placeholder:text-[#6B7280]"
+                className="border-border bg-card text-foreground placeholder:text-muted-foreground w-28 rounded-full border px-4 py-2 text-sm outline-none sm:w-auto"
                 autoFocus
               />
             ) : (
               <button
-                className="shrink-0 rounded-full border border-[#E5E5E5] bg-white px-4 py-2 text-sm whitespace-nowrap text-[#18181B] hover:bg-[#F5F5F5] dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white dark:hover:bg-[#383838]"
+                className="border-border bg-card text-foreground hover:bg-accent shrink-0 rounded-full border px-4 py-2 text-sm whitespace-nowrap"
                 onClick={() => {
                   setIsCreatingFolder(true);
                   setTimeout(() => newFolderInputRef.current?.focus(), 0);
@@ -529,7 +529,7 @@ function AgentSection({
             ))}
           {config.showNewAgentButton && (
             <button
-              className="bg-purple-30 hover:bg-violets-are-blue shrink-0 rounded-full px-4 py-2 text-sm whitespace-nowrap text-white"
+              className="bg-primary hover:bg-primary/90 shrink-0 rounded-full px-4 py-2 text-sm whitespace-nowrap text-white"
               onClick={() => {
                 setModalFolderId(currentFolderId);
                 setShowAgentTypeModal(true);
@@ -579,7 +579,7 @@ function AgentSection({
                 ))}
               </div>
             ) : hasNoAgentsAtAll && currentLevelFolders.length === 0 ? (
-              <div className="flex h-40 w-full flex-col items-center justify-center gap-3 text-[#71717A]">
+              <div className="text-muted-foreground flex h-40 w-full flex-col items-center justify-center gap-3">
                 <p>
                   {currentFolderId
                     ? t('agents.folders.empty')
@@ -587,7 +587,7 @@ function AgentSection({
                 </p>
                 {config.showNewAgentButton && !currentFolderId && (
                   <button
-                    className="bg-purple-30 hover:bg-violets-are-blue ml-2 rounded-full px-4 py-2 text-sm text-white"
+                    className="bg-primary hover:bg-primary/90 ml-2 rounded-full px-4 py-2 text-sm text-white"
                     onClick={() => {
                       setModalFolderId(currentFolderId);
                       setShowAgentTypeModal(true);
@@ -603,4 +603,4 @@ function AgentSection({
       </div>
     </div>
   );
-}
+}

frontend/src/agents/FolderCard.tsx

@@ -70,17 +70,15 @@ export default function FolderCard({
     <>
       <div
         className={`relative flex cursor-pointer items-center justify-between rounded-[1.2rem] px-4 py-3 sm:w-48 ${
-          isExpanded
-            ? 'bg-[#E5E5E5] dark:bg-[#454545]'
-            : 'bg-[#F6F6F6] hover:bg-[#ECECEC] dark:bg-[#383838] dark:hover:bg-[#383838]/80'
+          isExpanded ? 'bg-accent' : 'bg-muted hover:bg-accent'
         }`}
         onClick={() => onToggleExpand(folder.id)}
       >
         <div className="flex items-center gap-2 overflow-hidden">
-          <span className="truncate text-sm font-medium text-[#18181B] dark:text-[#E0E0E0]">
+          <span className="text-foreground truncate text-sm font-medium">
             {folder.name}
           </span>
-          <span className="shrink-0 text-xs text-[#71717A]">
+          <span className="text-muted-foreground shrink-0 text-xs">
             ({agentCount})
           </span>
         </div>

frontend/src/agents/NewAgent.tsx

@@ -73,6 +73,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
     token_limit: undefined,
     limited_request_mode: false,
     request_limit: undefined,
+    allow_system_prompt_override: false,
     models: [],
     default_model_id: '',
   });
@@ -241,6 +242,11 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
       formData.append('request_limit', '0');
     }
 
+    formData.append(
+      'allow_system_prompt_override',
+      agent.allow_system_prompt_override ? 'True' : 'False',
+    );
+
     if (imageFile) formData.append('image', imageFile);
 
     if (agent.tools && agent.tools.length > 0)
@@ -361,6 +367,11 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
       formData.append('request_limit', '0');
     }
 
+    formData.append(
+      'allow_system_prompt_override',
+      agent.allow_system_prompt_override ? 'True' : 'False',
+    );
+
     if (agent.models && agent.models.length > 0) {
       formData.append('models', JSON.stringify(agent.models));
     }
@@ -677,17 +688,17 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
     <div className="flex flex-col px-4 pt-4 pb-2 max-[1179px]:min-h-dvh min-[1180px]:h-dvh md:px-12 md:pt-12 md:pb-3">
       <div className="flex items-center gap-3 px-4">
         <button
-          className="rounded-full border p-3 text-sm text-gray-400 dark:border-0 dark:bg-[#28292D] dark:text-gray-500 dark:hover:bg-[#2E2F34]"
+          className="border-border text-muted-foreground hover:bg-accent rounded-full border p-3 text-sm"
           onClick={handleCancel}
         >
           <img src={ArrowLeft} alt="left-arrow" className="h-3 w-3" />
         </button>
-        <p className="text-eerie-black dark:text-bright-gray mt-px text-sm font-semibold">
+        <p className="text-foreground dark:text-foreground mt-px text-sm font-semibold">
           {t('agents.backToAll')}
         </p>
       </div>
       <div className="mt-5 flex w-full flex-wrap items-center justify-between gap-2 px-4">
-        <h1 className="text-eerie-black m-0 text-[32px] font-bold lg:text-[40px] dark:text-white">
+        <h1 className="text-foreground m-0 text-[32px] font-bold lg:text-[40px] dark:text-white">
           {modeConfig[effectiveMode].heading}
         </h1>
         {agent.agent_type === 'workflow' && (
@@ -697,14 +708,14 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
         )}
         <div className="flex flex-wrap items-center gap-1">
           <button
-            className="text-purple-30 dark:text-light-gray mr-4 rounded-3xl py-2 text-sm font-medium dark:bg-transparent"
+            className="text-primary dark:text-foreground mr-4 rounded-3xl py-2 text-sm font-medium"
             onClick={handleCancel}
           >
             {t('agents.form.buttons.cancel')}
           </button>
           {modeConfig[effectiveMode].showDelete && agent.id && (
             <button
-              className="group border-red-2000 text-red-2000 hover:bg-red-2000 flex items-center gap-2 rounded-3xl border border-solid px-5 py-2 text-sm font-medium transition-colors hover:text-white"
+              className="group border-destructive text-destructive hover:bg-destructive flex items-center gap-2 rounded-3xl border border-solid px-5 py-2 text-sm font-medium transition-colors hover:text-white"
               onClick={() => setDeleteConfirmation('ACTIVE')}
             >
               <span className="block h-4 w-4 bg-[url('/src/assets/red-trash.svg')] bg-contain bg-center bg-no-repeat transition-all group-hover:bg-[url('/src/assets/white-trash.svg')]" />
@@ -714,7 +725,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
           {modeConfig[effectiveMode].showSaveDraft && (
             <button
               disabled={isJsonSchemaInvalid()}
-              className={`border-violets-are-blue text-violets-are-blue hover:bg-violets-are-blue flex min-w-28 items-center justify-center rounded-3xl border border-solid px-5 py-2 text-sm font-medium whitespace-nowrap transition-colors hover:text-white ${
+              className={`border-primary text-primary hover:bg-primary/90 flex min-w-28 items-center justify-center rounded-3xl border border-solid px-5 py-2 text-sm font-medium whitespace-nowrap transition-colors hover:text-white ${
                 isJsonSchemaInvalid() ? 'cursor-not-allowed opacity-30' : ''
               }`}
               onClick={handleSaveDraft}
@@ -730,7 +741,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
           )}
           {modeConfig[effectiveMode].showAccessDetails && (
             <button
-              className="group border-violets-are-blue text-violets-are-blue hover:bg-violets-are-blue flex items-center gap-2 rounded-3xl border border-solid px-5 py-2 text-sm font-medium transition-colors hover:text-white"
+              className="group border-primary text-primary hover:bg-primary/90 flex items-center gap-2 rounded-3xl border border-solid px-5 py-2 text-sm font-medium transition-colors hover:text-white"
               onClick={() => navigate(`/agents/logs/${agent.id}`)}
             >
               <span className="block h-5 w-5 bg-[url('/src/assets/monitoring-purple.svg')] bg-contain bg-center bg-no-repeat transition-all group-hover:bg-[url('/src/assets/monitoring-white.svg')]" />
@@ -739,7 +750,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
           )}
           {modeConfig[effectiveMode].showAccessDetails && (
             <button
-              className="hover:bg-vi</button>olets-are-blue border-violets-are-blue text-violets-are-blue hover:bg-violets-are-blue rounded-3xl border border-solid px-5 py-2 text-sm font-medium transition-colors hover:text-white"
+              className="border-primary text-primary hover:bg-primary/90 rounded-3xl border border-solid px-5 py-2 text-sm font-medium transition-colors hover:text-white"
               onClick={() => setAgentDetails('ACTIVE')}
             >
               {t('agents.form.buttons.accessDetails')}
@@ -747,7 +758,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
           )}
           <button
             disabled={!isPublishable() || !hasChanges}
-            className={`${!isPublishable() || !hasChanges ? 'cursor-not-allowed opacity-30' : ''} bg-purple-30 hover:bg-violets-are-blue flex min-w-28 items-center justify-center rounded-3xl px-5 py-2 text-sm font-medium whitespace-nowrap text-white`}
+            className={`${!isPublishable() || !hasChanges ? 'cursor-not-allowed opacity-30' : ''} bg-primary hover:bg-primary/90 flex min-w-28 items-center justify-center rounded-3xl px-5 py-2 text-sm font-medium whitespace-nowrap text-white`}
             onClick={handlePublish}
           >
             <span className="flex items-center justify-center transition-all duration-200">
@@ -760,21 +771,21 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
           </button>
         </div>
       </div>
-      <div className="mt-3 flex w-full flex-1 grid-cols-5 flex-col gap-10 rounded-[30px] bg-[#F6F6F6] p-5 max-[1179px]:overflow-visible min-[1180px]:grid min-[1180px]:gap-5 min-[1180px]:overflow-hidden dark:bg-[#383838]">
+      <div className="bg-muted dark:bg-background mt-3 flex w-full flex-1 grid-cols-5 flex-col gap-10 rounded-[30px] p-5 max-[1179px]:overflow-visible min-[1180px]:grid min-[1180px]:gap-5 min-[1180px]:overflow-hidden">
         <div className="scrollbar-overlay col-span-2 flex flex-col gap-5 max-[1179px]:overflow-visible min-[1180px]:max-h-full min-[1180px]:overflow-y-auto min-[1180px]:pr-3">
-          <div className="dark:bg-raisin-black rounded-[30px] bg-white px-6 py-3 dark:text-[#E0E0E0]">
+          <div className="bg-card rounded-[30px] px-6 py-3">
             <h2 className="text-lg font-semibold">
               {t('agents.form.sections.meta')}
             </h2>
             <input
-              className="border-silver text-jet dark:bg-raisin-black dark:text-bright-gray dark:placeholder:text-silver mt-3 w-full rounded-3xl border bg-white px-5 py-3 text-sm outline-hidden placeholder:text-gray-400 dark:border-[#7E7E7E]"
+              className="border-border text-foreground dark:text-foreground dark:placeholder:text-silver bg-card dark:border-border mt-3 w-full rounded-3xl border px-5 py-3 text-sm outline-hidden placeholder:text-gray-400"
               type="text"
               value={agent.name}
               placeholder={t('agents.form.placeholders.agentName')}
               onChange={(e) => setAgent({ ...agent, name: e.target.value })}
             />
             <textarea
-              className="border-silver text-jet dark:bg-raisin-black dark:text-bright-gray dark:placeholder:text-silver mt-3 h-32 w-full rounded-xl border bg-white px-5 py-4 text-sm outline-hidden placeholder:text-gray-400 dark:border-[#7E7E7E]"
+              className="border-border text-foreground dark:text-foreground dark:placeholder:text-silver bg-card dark:border-border mt-3 h-32 w-full rounded-xl border px-5 py-4 text-sm outline-hidden placeholder:text-gray-400"
               placeholder={t('agents.form.placeholders.describeAgent')}
               value={agent.description}
               onChange={(e) =>
@@ -784,7 +795,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
             <div className="mt-3">
               <FileUpload
                 showPreview
-                className="dark:bg-raisin-black"
+                className="bg-card"
                 onUpload={handleUpload}
                 onRemove={() => setImageFile(null)}
                 uploadText={[
@@ -800,7 +811,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
               />
             </div>
           </div>
-          <div className="dark:bg-raisin-black rounded-[30px] bg-white px-6 py-3 dark:text-[#E0E0E0]">
+          <div className="bg-card rounded-[30px] px-6 py-3">
             <h2 className="text-lg font-semibold">
               {t('agents.form.sections.source')}
             </h2>
@@ -809,10 +820,10 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                 <button
                   ref={sourceAnchorButtonRef}
                   onClick={() => setIsSourcePopupOpen(!isSourcePopupOpen)}
-                  className={`border-silver dark:bg-raisin-black w-full truncate rounded-3xl border bg-white px-5 py-3 text-left text-sm dark:border-[#7E7E7E] ${
+                  className={`border-border bg-card dark:border-border w-full truncate rounded-3xl border px-5 py-3 text-left text-sm ${
                     selectedSourceIds.size > 0
-                      ? 'text-jet dark:text-bright-gray'
-                      : 'dark:text-silver text-gray-400'
+                      ? 'text-foreground dark:text-foreground'
+                      : 'dark:text-muted-foreground text-gray-400'
                   }`}
                 >
                   {selectedSourceIds.size > 0
@@ -892,17 +903,13 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                   }
                   size="w-full"
                   rounded="3xl"
-                  border="border"
-                  buttonClassName="bg-white dark:bg-[#222327] border-silver dark:border-[#7E7E7E]"
-                  optionsClassName="bg-white dark:bg-[#383838] border-silver dark:border-[#7E7E7E]"
                   placeholder={t('agents.form.placeholders.chunksPerQuery')}
-                  placeholderClassName="text-gray-400 dark:text-silver"
                   contentSize="text-sm"
                 />
               </div>
             </div>
           </div>
-          <div className="dark:bg-raisin-black rounded-[30px] bg-white px-6 py-3 dark:text-[#E0E0E0]">
+          <div className="bg-card rounded-[30px] px-6 py-3">
             <div className="flex flex-wrap items-end gap-1">
               <div className="min-w-20 grow basis-full sm:basis-0">
                 <Prompts
@@ -920,30 +927,24 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                   }
                   setPrompts={(newPrompts) => dispatch(setPrompts(newPrompts))}
                   title={t('agents.form.sections.prompt')}
-                  titleClassName="text-lg font-semibold dark:text-[#E0E0E0]"
+                  titleClassName="text-lg font-semibold"
                   showAddButton={false}
                   dropdownProps={{
                     size: 'w-full',
                     rounded: '3xl',
-                    border: 'border',
-                    buttonClassName:
-                      'bg-white dark:bg-[#222327] border-silver dark:border-[#7E7E7E]',
-                    optionsClassName:
-                      'bg-white dark:bg-[#383838] border-silver dark:border-[#7E7E7E]',
-                    placeholderClassName: 'text-gray-400 dark:text-silver',
                     contentSize: 'text-sm',
                   }}
                 />
               </div>
               <button
-                className="border-violets-are-blue text-violets-are-blue hover:bg-violets-are-blue min-w-20 shrink-0 basis-full rounded-3xl border-2 border-solid px-5 py-[11px] text-sm whitespace-nowrap transition-colors hover:text-white sm:basis-auto"
+                className="border-primary text-primary hover:bg-primary/90 min-w-20 shrink-0 basis-full rounded-3xl border border-solid px-5 py-3 text-sm whitespace-nowrap transition-colors hover:text-white sm:basis-auto"
                 onClick={() => setAddPromptModal('ACTIVE')}
               >
                 {t('agents.form.buttons.add')}
               </button>
             </div>
           </div>
-          <div className="dark:bg-raisin-black rounded-[30px] bg-white px-6 py-3 dark:text-[#E0E0E0]">
+          <div className="bg-card rounded-[30px] px-6 py-3">
             <h2 className="text-lg font-semibold">
               {t('agents.form.sections.tools')}
             </h2>
@@ -951,10 +952,10 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
               <button
                 ref={toolAnchorButtonRef}
                 onClick={() => setIsToolsPopupOpen(!isToolsPopupOpen)}
-                className={`border-silver dark:bg-raisin-black w-full truncate rounded-3xl border bg-white px-5 py-3 text-left text-sm dark:border-[#7E7E7E] ${
+                className={`border-border bg-card dark:border-border w-full truncate rounded-3xl border px-5 py-3 text-left text-sm ${
                   selectedTools.length > 0
-                    ? 'text-jet dark:text-bright-gray'
-                    : 'dark:text-silver text-gray-400'
+                    ? 'text-foreground dark:text-foreground'
+                    : 'dark:text-muted-foreground text-gray-400'
                 }`}
               >
                 {selectedTools.length > 0
@@ -992,7 +993,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
               />
             </div>
           </div>
-          <div className="dark:bg-raisin-black rounded-[30px] bg-white px-6 py-3 dark:text-[#E0E0E0]">
+          <div className="bg-card rounded-[30px] px-6 py-3">
             <h2 className="text-lg font-semibold">
               {t('agents.form.sections.agentType')}
             </h2>
@@ -1010,16 +1011,12 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                 }
                 size="w-full"
                 rounded="3xl"
-                border="border"
-                buttonClassName="bg-white dark:bg-[#222327] border-silver dark:border-[#7E7E7E]"
-                optionsClassName="bg-white dark:bg-[#383838] border-silver dark:border-[#7E7E7E]"
                 placeholder={t('agents.form.placeholders.selectType')}
-                placeholderClassName="text-gray-400 dark:text-silver"
                 contentSize="text-sm"
               />
             </div>
           </div>
-          <div className="dark:bg-raisin-black rounded-[30px] bg-white px-6 py-3 dark:text-[#E0E0E0]">
+          <div className="bg-card rounded-[30px] px-6 py-3">
             <h2 className="text-lg font-semibold">
               {t('agents.form.sections.models')}
             </h2>
@@ -1027,10 +1024,10 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
               <button
                 ref={modelAnchorButtonRef}
                 onClick={() => setIsModelsPopupOpen(!isModelsPopupOpen)}
-                className={`border-silver dark:bg-raisin-black w-full truncate rounded-3xl border bg-white px-5 py-3 text-left text-sm dark:border-[#7E7E7E] ${
+                className={`border-border bg-card dark:border-border w-full truncate rounded-3xl border px-5 py-3 text-left text-sm ${
                   selectedModelIds.size > 0
-                    ? 'text-jet dark:text-bright-gray'
-                    : 'dark:text-silver text-gray-400'
+                    ? 'text-foreground dark:text-foreground'
+                    : 'dark:text-muted-foreground text-gray-400'
                 }`}
               >
                 {selectedModelIds.size > 0
@@ -1082,20 +1079,16 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                     }
                     size="w-full"
                     rounded="3xl"
-                    border="border"
-                    buttonClassName="bg-white dark:bg-[#222327] border-silver dark:border-[#7E7E7E]"
-                    optionsClassName="bg-white dark:bg-[#383838] border-silver dark:border-[#7E7E7E]"
                     placeholder={t(
                       'agents.form.placeholders.selectDefaultModel',
                     )}
-                    placeholderClassName="text-gray-400 dark:text-silver"
                     contentSize="text-sm"
                   />
                 </div>
               )}
             </div>
           </div>
-          <div className="dark:bg-raisin-black rounded-[30px] bg-white px-6 py-3 dark:text-[#E0E0E0]">
+          <div className="bg-card rounded-[30px] px-6 py-3">
             <button
               onClick={() =>
                 setIsAdvancedSectionExpanded(!isAdvancedSectionExpanded)
@@ -1148,7 +1141,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
   "additionalProperties": false
 }`}
                   rows={9}
-                  className={`border-silver text-jet dark:bg-raisin-black dark:text-bright-gray mt-2 w-full rounded-2xl border bg-white px-4 py-3 font-mono text-sm outline-hidden dark:border-[#7E7E7E]`}
+                  className={`border-border text-foreground dark:text-foreground bg-card dark:border-border mt-2 w-full rounded-2xl border px-4 py-3 font-mono text-sm outline-hidden`}
                 />
                 {jsonSchemaText.trim() !== '' && (
                   <div
@@ -1194,7 +1187,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                       }}
                       className={`relative h-6 w-11 rounded-full transition-colors ${
                         agent.limited_token_mode
-                          ? 'bg-purple-30'
+                          ? 'bg-primary'
                           : 'bg-gray-300 dark:bg-gray-600'
                       }`}
                     >
@@ -1219,7 +1212,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                     }
                     disabled={!agent.limited_token_mode}
                     placeholder={t('agents.form.placeholders.enterTokenLimit')}
-                    className={`border-silver text-jet dark:bg-raisin-black dark:text-bright-gray dark:placeholder:text-silver mt-2 w-full rounded-3xl border bg-white px-5 py-3 text-sm outline-hidden placeholder:text-gray-400 dark:border-[#7E7E7E] ${
+                    className={`border-border text-foreground dark:text-foreground dark:placeholder:text-silver bg-card dark:border-border mt-2 w-full rounded-3xl border px-5 py-3 text-sm outline-hidden placeholder:text-gray-400 ${
                       !agent.limited_token_mode
                         ? 'cursor-not-allowed opacity-50'
                         : ''
@@ -1250,7 +1243,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                       }}
                       className={`relative h-6 w-11 rounded-full transition-colors ${
                         agent.limited_request_mode
-                          ? 'bg-purple-30'
+                          ? 'bg-primary'
                           : 'bg-gray-300 dark:bg-gray-600'
                       }`}
                     >
@@ -1277,18 +1270,55 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
                     placeholder={t(
                       'agents.form.placeholders.enterRequestLimit',
                     )}
-                    className={`border-silver text-jet dark:bg-raisin-black dark:text-bright-gray dark:placeholder:text-silver mt-2 w-full rounded-3xl border bg-white px-5 py-3 text-sm outline-hidden placeholder:text-gray-400 dark:border-[#7E7E7E] ${
+                    className={`border-border text-foreground dark:text-foreground dark:placeholder:text-silver bg-card dark:border-border mt-2 w-full rounded-3xl border px-5 py-3 text-sm outline-hidden placeholder:text-gray-400 ${
                       !agent.limited_request_mode
                         ? 'cursor-not-allowed opacity-50'
                         : ''
                     }`}
                   />
                 </div>
+
+                <div className="mt-6">
+                  <div className="flex items-center justify-between gap-4">
+                    <div className="min-w-0 flex-1">
+                      <h2 className="text-sm font-medium">
+                        {t('agents.form.advanced.systemPromptOverride')}
+                      </h2>
+                      <p className="mt-1 text-xs text-gray-600 dark:text-gray-400">
+                        {t(
+                          'agents.form.advanced.systemPromptOverrideDescription',
+                        )}
+                      </p>
+                    </div>
+                    <button
+                      onClick={() =>
+                        setAgent({
+                          ...agent,
+                          allow_system_prompt_override:
+                            !agent.allow_system_prompt_override,
+                        })
+                      }
+                      className={`relative h-6 w-11 shrink-0 rounded-full transition-colors ${
+                        agent.allow_system_prompt_override
+                          ? 'bg-primary'
+                          : 'bg-gray-300 dark:bg-gray-600'
+                      }`}
+                    >
+                      <span
+                        className={`absolute top-0.5 h-5 w-5 transform rounded-full bg-white transition-transform ${
+                          agent.allow_system_prompt_override
+                            ? ''
+                            : '-translate-x-5'
+                        }`}
+                      />
+                    </button>
+                  </div>
+                </div>
               </div>
             )}
           </div>
         </div>
-        <div className="col-span-3 flex flex-col gap-2 max-[1179px]:h-auto max-[1179px]:px-0 max-[1179px]:py-0 min-[1180px]:h-full min-[1180px]:py-2 dark:text-[#E0E0E0]">
+        <div className="col-span-3 flex flex-col gap-2 max-[1179px]:h-auto max-[1179px]:px-0 max-[1179px]:py-0 min-[1180px]:h-full min-[1180px]:py-2">
           <h2 className="text-lg font-semibold">
             {t('agents.form.sections.preview')}
           </h2>
@@ -1331,7 +1361,7 @@ function AgentPreviewArea() {
   const { t } = useTranslation();
   const selectedAgent = useSelector(selectSelectedAgent);
   return (
-    <div className="dark:bg-raisin-black w-full rounded-[30px] border border-[#F6F6F6] bg-white max-[1179px]:h-[600px] min-[1180px]:h-full dark:border-[#7E7E7E]">
+    <div className="bg-card border-border w-full rounded-[30px] border max-[1179px]:h-[600px] min-[1180px]:h-full">
       {selectedAgent?.status === 'published' ? (
         <div className="flex h-full w-full flex-col overflow-hidden rounded-[30px]">
           <AgentPreview />
@@ -1339,7 +1369,7 @@ function AgentPreviewArea() {
       ) : (
         <div className="flex h-full w-full flex-col items-center justify-center gap-2">
           <span className="block h-12 w-12 bg-[url('/src/assets/science-spark.svg')] bg-contain bg-center bg-no-repeat transition-all dark:bg-[url('/src/assets/science-spark-dark.svg')]" />{' '}
-          <p className="dark:text-gray-4000 text-xs text-[#18181B]">
+          <p className="text-muted-foreground text-xs">
             {t('agents.form.preview.publishedPreview')}
           </p>
         </div>

frontend/src/agents/SharedAgent.tsx

@@ -143,7 +143,7 @@ export default function SharedAgent() {
             alt="No agent found"
             className="mx-auto mb-6 h-32 w-32"
           />
-          <p className="dark:text-gray-4000 text-center text-lg text-[#71717A]">
+          <p className="text-muted-foreground text-center text-lg">
             {t('agents.shared.notFound')}
           </p>
         </div>
@@ -157,7 +157,7 @@ export default function SharedAgent() {
           alt="agent-logo"
           className="h-6 w-6 rounded-full object-contain"
         />
-        <h2 className="text-eerie-black text-lg font-semibold dark:text-[#E0E0E0]">
+        <h2 className="text-foreground text-lg font-semibold">
           {sharedAgent.name}
         </h2>
       </div>
@@ -186,7 +186,7 @@ export default function SharedAgent() {
               autoFocus={false}
             />
           </div>
-          <p className="text-gray-4000 dark:text-sonic-silver hidden w-screen self-center bg-transparent py-2 text-center text-xs md:inline md:w-full">
+          <p className="text-muted-foreground hidden w-screen self-center bg-transparent py-2 text-center text-xs md:inline md:w-full">
             {t('tagline')}
           </p>
         </div>

frontend/src/agents/SharedAgentCard.tsx

@@ -10,7 +10,7 @@ export default function SharedAgentCard({ agent }: { agent: Agent }) {
     agent.shared_metadata !== null &&
     Object.keys(agent.shared_metadata).length > 0;
   return (
-    <div className="border-dark-gray dark:border-grey flex w-full max-w-[720px] flex-col rounded-3xl border p-6 shadow-xs sm:w-fit sm:min-w-[480px]">
+    <div className="border-border dark:border-border flex w-full max-w-[720px] flex-col rounded-3xl border p-6 shadow-xs sm:w-fit sm:min-w-[480px]">
       <div className="flex items-center gap-3">
         <div className="flex h-12 w-12 items-center justify-center overflow-hidden rounded-full p-1">
           <AgentImage
@@ -19,10 +19,10 @@ export default function SharedAgentCard({ agent }: { agent: Agent }) {
           />
         </div>
         <div className="flex max-h-[92px] w-[80%] flex-col gap-px">
-          <h2 className="text-eerie-black text-base font-semibold sm:text-lg dark:text-[#E0E0E0]">
+          <h2 className="text-foreground text-base font-semibold sm:text-lg">
             {agent.name}
           </h2>
-          <p className="dark:text-gray-4000 overflow-y-auto text-xs text-wrap break-all text-[#71717A] sm:text-sm">
+          <p className="text-muted-foreground overflow-y-auto text-xs text-wrap break-all sm:text-sm">
             {agent.description}
           </p>
         </div>
@@ -30,12 +30,12 @@ export default function SharedAgentCard({ agent }: { agent: Agent }) {
       {hasSharedMetadata && (
         <div className="mt-4 flex items-center gap-8">
           {agent.shared_metadata?.shared_by && (
-            <p className="text-eerie-black text-xs font-light sm:text-sm dark:text-[#E0E0E0]">
+            <p className="text-foreground text-xs font-light sm:text-sm">
               by {agent.shared_metadata.shared_by}
             </p>
           )}
           {agent.shared_metadata?.shared_at && (
-            <p className="dark:text-gray-4000 text-xs font-light text-[#71717A] sm:text-sm">
+            <p className="text-muted-foreground text-xs font-light sm:text-sm">
               Shared on{' '}
               {new Date(agent.shared_metadata.shared_at).toLocaleString(
                 'en-US',
@@ -54,14 +54,14 @@ export default function SharedAgentCard({ agent }: { agent: Agent }) {
       )}
       {agent.tool_details && agent.tool_details.length > 0 && (
         <div className="mt-8">
-          <p className="text-eerie-black text-sm font-semibold sm:text-base dark:text-[#E0E0E0]">
+          <p className="text-foreground text-sm font-semibold sm:text-base">
             Connected Tools
           </p>
           <div className="mt-2 flex flex-wrap gap-2">
             {agent.tool_details.map((tool, index) => (
               <span
                 key={index}
-                className="bg-bright-gray text-eerie-black dark:bg-dark-charcoal flex items-center gap-1 rounded-full px-3 py-1 text-xs font-light dark:text-[#E0E0E0]"
+                className="bg-accent text-foreground dark:bg-card flex items-center gap-1 rounded-full px-3 py-1 text-xs font-light"
               >
                 <img
                   src={`/toolIcons/tool_${tool.name}.svg`}