From 3f7de867cc3f38177e10c90d15b9069b33052b34 Mon Sep 17 00:00:00 2001 From: Siddhant Rai <47355538+siiddhantt@users.noreply.github.com> Date: Fri, 14 Nov 2025 16:43:19 +0530 Subject: [PATCH 01/93] feat: model registry and capabilities for multi-provider support (#2158) * feat: Implement model registry and capabilities for multi-provider support - Added ModelRegistry to manage available models and their capabilities. - Introduced ModelProvider enum for different LLM providers. - Created ModelCapabilities dataclass to define model features. - Implemented methods to load models based on API keys and settings. - Added utility functions for model management in model_utils.py. - Updated settings.py to include provider-specific API keys. - Refactored LLM classes (Anthropic, OpenAI, Google, etc.) to utilize new model registry. - Enhanced utility functions to handle token limits and model validation. - Improved code structure and logging for better maintainability. * feat: Add model selection feature with API integration and UI component * feat: Add model selection and default model functionality in agent management * test: Update assertions and formatting in stream processing tests * refactor(llm): Standardize model identifier to model_id * fix tests --------- Co-authored-by: Alex --- application/agents/agent_creator.py | 4 + application/agents/base.py | 7 +- application/agents/react_agent.py | 4 +- application/api/answer/routes/answer.py | 5 + application/api/answer/routes/base.py | 45 ++-- application/api/answer/routes/stream.py | 5 + .../answer/services/conversation_service.py | 9 +- .../api/answer/services/stream_processor.py | 48 +++- application/api/user/agents/routes.py | 31 ++- application/api/user/models/__init__.py | 3 + application/api/user/models/routes.py | 25 ++ application/api/user/routes.py | 4 + application/core/model_configs.py | 223 +++++++++++++++++ application/core/model_settings.py | 236 ++++++++++++++++++ application/core/model_utils.py | 91 +++++++ application/core/settings.py | 25 +- application/llm/anthropic.py | 40 ++- application/llm/base.py | 34 +-- application/llm/docsgpt_provider.py | 14 +- application/llm/google_ai.py | 42 +--- application/llm/groq.py | 11 +- application/llm/handlers/base.py | 4 +- application/llm/llm_creator.py | 36 ++- application/llm/openai.py | 45 ++-- application/retriever/classic_rag.py | 6 +- application/utils.py | 78 +++--- application/worker.py | 4 +- frontend/src/Hero.tsx | 10 +- frontend/src/agents/NewAgent.tsx | 136 ++++++++++ frontend/src/agents/agentPreviewSlice.ts | 13 +- frontend/src/agents/types/index.ts | 2 + frontend/src/api/endpoints.ts | 1 + frontend/src/api/services/modelService.ts | 25 ++ frontend/src/assets/rounded-tick.svg | 3 + frontend/src/components/DropdownModel.tsx | 138 ++++++++++ .../src/conversation/conversationHandlers.ts | 10 + .../src/conversation/conversationModels.ts | 1 + .../src/conversation/conversationSlice.ts | 9 +- frontend/src/locale/en.json | 11 + frontend/src/locale/es.json | 11 + frontend/src/locale/jp.json | 11 + frontend/src/locale/ru.json | 11 + frontend/src/locale/zh-TW.json | 11 + frontend/src/locale/zh.json | 11 + frontend/src/models/types.ts | 25 ++ frontend/src/preferences/preferenceSlice.ts | 40 ++- frontend/src/store.ts | 4 + tests/agents/test_agent_creator.py | 2 +- tests/agents/test_base_agent.py | 4 +- tests/agents/test_react_agent.py | 2 +- tests/api/answer/routes/test_base.py | 9 +- .../services/test_conversation_service.py | 10 +- tests/conftest.py | 2 +- tests/llm/test_anthropic_llm.py | 23 +- 54 files changed, 1388 insertions(+), 226 deletions(-) create mode 100644 application/api/user/models/__init__.py create mode 100644 application/api/user/models/routes.py create mode 100644 application/core/model_configs.py create mode 100644 application/core/model_settings.py create mode 100644 application/core/model_utils.py create mode 100644 frontend/src/api/services/modelService.ts create mode 100644 frontend/src/assets/rounded-tick.svg create mode 100644 frontend/src/components/DropdownModel.tsx create mode 100644 frontend/src/models/types.ts diff --git a/application/agents/agent_creator.py b/application/agents/agent_creator.py index bf37d4ec..44e89552 100644 --- a/application/agents/agent_creator.py +++ b/application/agents/agent_creator.py @@ -1,5 +1,8 @@ from application.agents.classic_agent import ClassicAgent from application.agents.react_agent import ReActAgent +import logging + +logger = logging.getLogger(__name__) class AgentCreator: @@ -13,4 +16,5 @@ class AgentCreator: agent_class = cls.agents.get(type.lower()) if not agent_class: raise ValueError(f"No agent class found for type {type}") + return agent_class(*args, **kwargs) diff --git a/application/agents/base.py b/application/agents/base.py index 27428fc3..dbf15a1f 100644 --- a/application/agents/base.py +++ b/application/agents/base.py @@ -21,7 +21,7 @@ class BaseAgent(ABC): self, endpoint: str, llm_name: str, - gpt_model: str, + model_id: str, api_key: str, user_api_key: Optional[str] = None, prompt: str = "", @@ -37,7 +37,7 @@ class BaseAgent(ABC): ): self.endpoint = endpoint self.llm_name = llm_name - self.gpt_model = gpt_model + self.model_id = model_id self.api_key = api_key self.user_api_key = user_api_key self.prompt = prompt @@ -52,6 +52,7 @@ class BaseAgent(ABC): api_key=api_key, user_api_key=user_api_key, decoded_token=decoded_token, + model_id=model_id, ) self.retrieved_docs = retrieved_docs or [] self.llm_handler = LLMHandlerCreator.create_handler( @@ -316,7 +317,7 @@ class BaseAgent(ABC): return messages def _llm_gen(self, messages: List[Dict], log_context: Optional[LogContext] = None): - gen_kwargs = {"model": self.gpt_model, "messages": messages} + gen_kwargs = {"model": self.model_id, "messages": messages} if ( hasattr(self.llm, "_supports_tools") diff --git a/application/agents/react_agent.py b/application/agents/react_agent.py index 49dd29d8..116fa4aa 100644 --- a/application/agents/react_agent.py +++ b/application/agents/react_agent.py @@ -86,7 +86,7 @@ class ReActAgent(BaseAgent): messages = [{"role": "user", "content": plan_prompt}] plan_stream = self.llm.gen_stream( - model=self.gpt_model, + model=self.model_id, messages=messages, tools=self.tools if self.tools else None, ) @@ -151,7 +151,7 @@ class ReActAgent(BaseAgent): messages = [{"role": "user", "content": final_prompt}] final_stream = self.llm.gen_stream( - model=self.gpt_model, messages=messages, tools=None + model=self.model_id, messages=messages, tools=None ) if log_context: diff --git a/application/api/answer/routes/answer.py b/application/api/answer/routes/answer.py index 87d80059..bc7ec58c 100644 --- a/application/api/answer/routes/answer.py +++ b/application/api/answer/routes/answer.py @@ -54,6 +54,10 @@ class AnswerResource(Resource, BaseAnswerResource): default=True, description="Whether to save the conversation", ), + "model_id": fields.String( + required=False, + description="Model ID to use for this request", + ), "passthrough": fields.Raw( required=False, description="Dynamic parameters to inject into prompt template", @@ -97,6 +101,7 @@ class AnswerResource(Resource, BaseAnswerResource): isNoneDoc=data.get("isNoneDoc"), index=None, should_save_conversation=data.get("save_conversation", True), + model_id=processor.model_id, ) stream_result = self.process_response_stream(stream) diff --git a/application/api/answer/routes/base.py b/application/api/answer/routes/base.py index 43e83ed2..aefb66c6 100644 --- a/application/api/answer/routes/base.py +++ b/application/api/answer/routes/base.py @@ -7,11 +7,16 @@ from flask import jsonify, make_response, Response from flask_restx import Namespace from application.api.answer.services.conversation_service import ConversationService +from application.core.model_utils import ( + get_api_key_for_provider, + get_default_model_id, + get_provider_from_model_id, +) from application.core.mongo_db import MongoDB from application.core.settings import settings from application.llm.llm_creator import LLMCreator -from application.utils import check_required_fields, get_gpt_model +from application.utils import check_required_fields logger = logging.getLogger(__name__) @@ -27,7 +32,7 @@ class BaseAnswerResource: db = mongo[settings.MONGO_DB_NAME] self.db = db self.user_logs_collection = db["user_logs"] - self.gpt_model = get_gpt_model() + self.default_model_id = get_default_model_id() self.conversation_service = ConversationService() def validate_request( @@ -54,7 +59,6 @@ class BaseAnswerResource: api_key = agent_config.get("user_api_key") if not api_key: return None - agents_collection = self.db["agents"] agent = agents_collection.find_one({"key": api_key}) @@ -62,7 +66,6 @@ class BaseAnswerResource: return make_response( jsonify({"success": False, "message": "Invalid API key."}), 401 ) - limited_token_mode_raw = agent.get("limited_token_mode", False) limited_request_mode_raw = agent.get("limited_request_mode", False) @@ -110,15 +113,12 @@ class BaseAnswerResource: daily_token_usage = token_result[0]["total_tokens"] if token_result else 0 else: daily_token_usage = 0 - if limited_request_mode: daily_request_usage = token_usage_collection.count_documents(match_query) else: daily_request_usage = 0 - if not limited_token_mode and not limited_request_mode: return None - token_exceeded = ( limited_token_mode and token_limit > 0 and daily_token_usage >= token_limit ) @@ -138,7 +138,6 @@ class BaseAnswerResource: ), 429, ) - return None def complete_stream( @@ -155,6 +154,7 @@ class BaseAnswerResource: agent_id: Optional[str] = None, is_shared_usage: bool = False, shared_token: Optional[str] = None, + model_id: Optional[str] = None, ) -> Generator[str, None, None]: """ Generator function that streams the complete conversation response. @@ -173,6 +173,7 @@ class BaseAnswerResource: agent_id: ID of agent used is_shared_usage: Flag for shared agent usage shared_token: Token for shared agent + model_id: Model ID used for the request retrieved_docs: Pre-fetched documents for sources (optional) Yields: @@ -220,7 +221,6 @@ class BaseAnswerResource: elif "type" in line: data = json.dumps(line) yield f"data: {data}\n\n" - if is_structured and structured_chunks: structured_data = { "type": "structured_answer", @@ -230,15 +230,22 @@ class BaseAnswerResource: } data = json.dumps(structured_data) yield f"data: {data}\n\n" - if isNoneDoc: for doc in source_log_docs: doc["source"] = "None" + provider = ( + get_provider_from_model_id(model_id) + if model_id + else settings.LLM_PROVIDER + ) + system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER) + llm = LLMCreator.create_llm( - settings.LLM_PROVIDER, - api_key=settings.API_KEY, + provider or settings.LLM_PROVIDER, + api_key=system_api_key, user_api_key=user_api_key, decoded_token=decoded_token, + model_id=model_id, ) if should_save_conversation: @@ -250,7 +257,7 @@ class BaseAnswerResource: source_log_docs, tool_calls, llm, - self.gpt_model, + model_id or self.default_model_id, decoded_token, index=index, api_key=user_api_key, @@ -280,12 +287,11 @@ class BaseAnswerResource: log_data["structured_output"] = True if schema_info: log_data["schema"] = schema_info - # Clean up text fields to be no longer than 10000 characters + for key, value in log_data.items(): if isinstance(value, str) and len(value) > 10000: log_data[key] = value[:10000] - self.user_logs_collection.insert_one(log_data) data = json.dumps({"type": "end"}) @@ -293,6 +299,7 @@ class BaseAnswerResource: except GeneratorExit: logger.info(f"Stream aborted by client for question: {question[:50]}... ") # Save partial response + if should_save_conversation and response_full: try: if isNoneDoc: @@ -312,7 +319,7 @@ class BaseAnswerResource: source_log_docs, tool_calls, llm, - self.gpt_model, + model_id or self.default_model_id, decoded_token, index=index, api_key=user_api_key, @@ -369,7 +376,7 @@ class BaseAnswerResource: thought = event["thought"] elif event["type"] == "error": logger.error(f"Error from stream: {event['error']}") - return None, None, None, None, event["error"] + return None, None, None, None, event["error"], None elif event["type"] == "end": stream_ended = True except (json.JSONDecodeError, KeyError) as e: @@ -377,8 +384,7 @@ class BaseAnswerResource: continue if not stream_ended: logger.error("Stream ended unexpectedly without an 'end' event.") - return None, None, None, None, "Stream ended unexpectedly" - + return None, None, None, None, "Stream ended unexpectedly", None result = ( conversation_id, response_full, @@ -390,7 +396,6 @@ class BaseAnswerResource: if is_structured: result = result + ({"structured": True, "schema": schema_info},) - return result def error_stream_generate(self, err_response): diff --git a/application/api/answer/routes/stream.py b/application/api/answer/routes/stream.py index 92e41c14..b2827a93 100644 --- a/application/api/answer/routes/stream.py +++ b/application/api/answer/routes/stream.py @@ -57,6 +57,10 @@ class StreamResource(Resource, BaseAnswerResource): default=True, description="Whether to save the conversation", ), + "model_id": fields.String( + required=False, + description="Model ID to use for this request", + ), "attachments": fields.List( fields.String, required=False, description="List of attachment IDs" ), @@ -101,6 +105,7 @@ class StreamResource(Resource, BaseAnswerResource): agent_id=data.get("agent_id"), is_shared_usage=processor.is_shared_usage, shared_token=processor.shared_token, + model_id=processor.model_id, ), mimetype="text/event-stream", ) diff --git a/application/api/answer/services/conversation_service.py b/application/api/answer/services/conversation_service.py index eca842d6..0e98983e 100644 --- a/application/api/answer/services/conversation_service.py +++ b/application/api/answer/services/conversation_service.py @@ -52,7 +52,7 @@ class ConversationService: sources: List[Dict[str, Any]], tool_calls: List[Dict[str, Any]], llm: Any, - gpt_model: str, + model_id: str, decoded_token: Dict[str, Any], index: Optional[int] = None, api_key: Optional[str] = None, @@ -66,7 +66,7 @@ class ConversationService: if not user_id: raise ValueError("User ID not found in token") current_time = datetime.now(timezone.utc) - + # clean up in sources array such that we save max 1k characters for text part for source in sources: if "text" in source and isinstance(source["text"], str): @@ -90,6 +90,7 @@ class ConversationService: f"queries.{index}.tool_calls": tool_calls, f"queries.{index}.timestamp": current_time, f"queries.{index}.attachments": attachment_ids, + f"queries.{index}.model_id": model_id, } }, ) @@ -120,6 +121,7 @@ class ConversationService: "tool_calls": tool_calls, "timestamp": current_time, "attachments": attachment_ids, + "model_id": model_id, } } }, @@ -146,7 +148,7 @@ class ConversationService: ] completion = llm.gen( - model=gpt_model, messages=messages_summary, max_tokens=30 + model=model_id, messages=messages_summary, max_tokens=30 ) conversation_data = { @@ -162,6 +164,7 @@ class ConversationService: "tool_calls": tool_calls, "timestamp": current_time, "attachments": attachment_ids, + "model_id": model_id, } ], } diff --git a/application/api/answer/services/stream_processor.py b/application/api/answer/services/stream_processor.py index bb890937..586e7696 100644 --- a/application/api/answer/services/stream_processor.py +++ b/application/api/answer/services/stream_processor.py @@ -12,12 +12,17 @@ from bson.objectid import ObjectId from application.agents.agent_creator import AgentCreator from application.api.answer.services.conversation_service import ConversationService from application.api.answer.services.prompt_renderer import PromptRenderer +from application.core.model_utils import ( + get_api_key_for_provider, + get_default_model_id, + get_provider_from_model_id, + validate_model_id, +) from application.core.mongo_db import MongoDB from application.core.settings import settings from application.retriever.retriever_creator import RetrieverCreator from application.utils import ( calculate_doc_token_budget, - get_gpt_model, limit_chat_history, ) @@ -83,7 +88,7 @@ class StreamProcessor: self.retriever_config = {} self.is_shared_usage = False self.shared_token = None - self.gpt_model = get_gpt_model() + self.model_id: Optional[str] = None self.conversation_service = ConversationService() self.prompt_renderer = PromptRenderer() self._prompt_content: Optional[str] = None @@ -91,6 +96,7 @@ class StreamProcessor: def initialize(self): """Initialize all required components for processing""" + self._validate_and_set_model() self._configure_agent() self._configure_source() self._configure_retriever() @@ -112,7 +118,7 @@ class StreamProcessor: ] else: self.history = limit_chat_history( - json.loads(self.data.get("history", "[]")), gpt_model=self.gpt_model + json.loads(self.data.get("history", "[]")), model_id=self.model_id ) def _process_attachments(self): @@ -143,6 +149,25 @@ class StreamProcessor: ) return attachments + def _validate_and_set_model(self): + """Validate and set model_id from request""" + from application.core.model_settings import ModelRegistry + + requested_model = self.data.get("model_id") + + if requested_model: + if not validate_model_id(requested_model): + registry = ModelRegistry.get_instance() + available_models = [m.id for m in registry.get_enabled_models()] + raise ValueError( + f"Invalid model_id '{requested_model}'. " + f"Available models: {', '.join(available_models[:5])}" + + (f" and {len(available_models) - 5} more" if len(available_models) > 5 else "") + ) + self.model_id = requested_model + else: + self.model_id = get_default_model_id() + def _get_agent_key(self, agent_id: Optional[str], user_id: Optional[str]) -> tuple: """Get API key for agent with access control""" if not agent_id: @@ -322,7 +347,7 @@ class StreamProcessor: def _configure_retriever(self): history_token_limit = int(self.data.get("token_limit", 2000)) doc_token_limit = calculate_doc_token_budget( - gpt_model=self.gpt_model, history_token_limit=history_token_limit + model_id=self.model_id, history_token_limit=history_token_limit ) self.retriever_config = { @@ -344,7 +369,7 @@ class StreamProcessor: prompt=get_prompt(self.agent_config["prompt_id"], self.prompts_collection), chunks=self.retriever_config["chunks"], doc_token_limit=self.retriever_config.get("doc_token_limit", 50000), - gpt_model=self.gpt_model, + model_id=self.model_id, user_api_key=self.agent_config["user_api_key"], decoded_token=self.decoded_token, ) @@ -626,12 +651,19 @@ class StreamProcessor: tools_data=tools_data, ) + provider = ( + get_provider_from_model_id(self.model_id) + if self.model_id + else settings.LLM_PROVIDER + ) + system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER) + return AgentCreator.create_agent( self.agent_config["agent_type"], endpoint="stream", - llm_name=settings.LLM_PROVIDER, - gpt_model=self.gpt_model, - api_key=settings.API_KEY, + llm_name=provider or settings.LLM_PROVIDER, + model_id=self.model_id, + api_key=system_api_key, user_api_key=self.agent_config["user_api_key"], prompt=rendered_prompt, chat_history=self.history, diff --git a/application/api/user/agents/routes.py b/application/api/user/agents/routes.py index 6c43342f..bcf68b56 100644 --- a/application/api/user/agents/routes.py +++ b/application/api/user/agents/routes.py @@ -95,6 +95,8 @@ class GetAgent(Resource): "shared": agent.get("shared_publicly", False), "shared_metadata": agent.get("shared_metadata", {}), "shared_token": agent.get("shared_token", ""), + "models": agent.get("models", []), + "default_model_id": agent.get("default_model_id", ""), } return make_response(jsonify(data), 200) except Exception as e: @@ -172,6 +174,8 @@ class GetAgents(Resource): "shared": agent.get("shared_publicly", False), "shared_metadata": agent.get("shared_metadata", {}), "shared_token": agent.get("shared_token", ""), + "models": agent.get("models", []), + "default_model_id": agent.get("default_model_id", ""), } for agent in agents if "source" in agent or "retriever" in agent @@ -230,6 +234,14 @@ class CreateAgent(Resource): required=False, description="Request limit for the agent in limited mode", ), + "models": fields.List( + fields.String, + required=False, + description="List of available model IDs for this agent", + ), + "default_model_id": fields.String( + required=False, description="Default model ID for this agent" + ), }, ) @@ -258,6 +270,11 @@ class CreateAgent(Resource): data["json_schema"] = json.loads(data["json_schema"]) except json.JSONDecodeError: data["json_schema"] = None + if "models" in data: + try: + data["models"] = json.loads(data["models"]) + except json.JSONDecodeError: + data["models"] = [] print(f"Received data: {data}") # Validate JSON schema if provided @@ -399,6 +416,8 @@ class CreateAgent(Resource): "updatedAt": datetime.datetime.now(datetime.timezone.utc), "lastUsedAt": None, "key": key, + "models": data.get("models", []), + "default_model_id": data.get("default_model_id", ""), } if new_agent["chunks"] == "": new_agent["chunks"] = "2" @@ -464,6 +483,14 @@ class UpdateAgent(Resource): required=False, description="Request limit for the agent in limited mode", ), + "models": fields.List( + fields.String, + required=False, + description="List of available model IDs for this agent", + ), + "default_model_id": fields.String( + required=False, description="Default model ID for this agent" + ), }, ) @@ -487,7 +514,7 @@ class UpdateAgent(Resource): data = request.get_json() else: data = request.form.to_dict() - json_fields = ["tools", "sources", "json_schema"] + json_fields = ["tools", "sources", "json_schema", "models"] for field in json_fields: if field in data and data[field]: try: @@ -555,6 +582,8 @@ class UpdateAgent(Resource): "token_limit", "limited_request_mode", "request_limit", + "models", + "default_model_id", ] for field in allowed_fields: diff --git a/application/api/user/models/__init__.py b/application/api/user/models/__init__.py new file mode 100644 index 00000000..f32afa11 --- /dev/null +++ b/application/api/user/models/__init__.py @@ -0,0 +1,3 @@ +from .routes import models_ns + +__all__ = ["models_ns"] diff --git a/application/api/user/models/routes.py b/application/api/user/models/routes.py new file mode 100644 index 00000000..886999b2 --- /dev/null +++ b/application/api/user/models/routes.py @@ -0,0 +1,25 @@ +from flask import current_app, jsonify, make_response +from flask_restx import Namespace, Resource + +from application.core.model_settings import ModelRegistry + +models_ns = Namespace("models", description="Available models", path="/api") + + +@models_ns.route("/models") +class ModelsListResource(Resource): + def get(self): + """Get list of available models with their capabilities.""" + try: + registry = ModelRegistry.get_instance() + models = registry.get_enabled_models() + + response = { + "models": [model.to_dict() for model in models], + "default_model_id": registry.default_model_id, + "count": len(models), + } + except Exception as err: + current_app.logger.error(f"Error fetching models: {err}", exc_info=True) + return make_response(jsonify({"success": False}), 500) + return make_response(jsonify(response), 200) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 1e0dbb4e..82e395c5 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -10,6 +10,7 @@ from .agents import agents_ns, agents_sharing_ns, agents_webhooks_ns from .analytics import analytics_ns from .attachments import attachments_ns from .conversations import conversations_ns +from .models import models_ns from .prompts import prompts_ns from .sharing import sharing_ns from .sources import sources_chunks_ns, sources_ns, sources_upload_ns @@ -27,6 +28,9 @@ api.add_namespace(attachments_ns) # Conversations api.add_namespace(conversations_ns) +# Models +api.add_namespace(models_ns) + # Agents (main, sharing, webhooks) api.add_namespace(agents_ns) api.add_namespace(agents_sharing_ns) diff --git a/application/core/model_configs.py b/application/core/model_configs.py new file mode 100644 index 00000000..b802ee27 --- /dev/null +++ b/application/core/model_configs.py @@ -0,0 +1,223 @@ +""" +Model configurations for all supported LLM providers. +""" + +from application.core.model_settings import ( + AvailableModel, + ModelCapabilities, + ModelProvider, +) + +OPENAI_ATTACHMENTS = [ + "application/pdf", + "image/png", + "image/jpeg", + "image/jpg", + "image/webp", + "image/gif", +] + +GOOGLE_ATTACHMENTS = [ + "application/pdf", + "image/png", + "image/jpeg", + "image/jpg", + "image/webp", + "image/gif", +] + + +OPENAI_MODELS = [ + AvailableModel( + id="gpt-4o", + provider=ModelProvider.OPENAI, + display_name="GPT-4 Omni", + description="Latest and most capable model", + capabilities=ModelCapabilities( + supports_tools=True, + supports_structured_output=True, + supported_attachment_types=OPENAI_ATTACHMENTS, + context_window=128000, + ), + ), + AvailableModel( + id="gpt-4o-mini", + provider=ModelProvider.OPENAI, + display_name="GPT-4 Omni Mini", + description="Fast and efficient", + capabilities=ModelCapabilities( + supports_tools=True, + supports_structured_output=True, + supported_attachment_types=OPENAI_ATTACHMENTS, + context_window=128000, + ), + ), + AvailableModel( + id="gpt-4-turbo", + provider=ModelProvider.OPENAI, + display_name="GPT-4 Turbo", + description="Fast GPT-4 with 128k context", + capabilities=ModelCapabilities( + supports_tools=True, + supports_structured_output=True, + supported_attachment_types=OPENAI_ATTACHMENTS, + context_window=128000, + ), + ), + AvailableModel( + id="gpt-4", + provider=ModelProvider.OPENAI, + display_name="GPT-4", + description="Most capable model", + capabilities=ModelCapabilities( + supports_tools=True, + supports_structured_output=True, + supported_attachment_types=OPENAI_ATTACHMENTS, + context_window=8192, + ), + ), + AvailableModel( + id="gpt-3.5-turbo", + provider=ModelProvider.OPENAI, + display_name="GPT-3.5 Turbo", + description="Fast and cost-effective", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=4096, + ), + ), +] + + +ANTHROPIC_MODELS = [ + AvailableModel( + id="claude-3-5-sonnet-20241022", + provider=ModelProvider.ANTHROPIC, + display_name="Claude 3.5 Sonnet (Latest)", + description="Latest Claude 3.5 Sonnet with enhanced capabilities", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=200000, + ), + ), + AvailableModel( + id="claude-3-5-sonnet", + provider=ModelProvider.ANTHROPIC, + display_name="Claude 3.5 Sonnet", + description="Balanced performance and capability", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=200000, + ), + ), + AvailableModel( + id="claude-3-opus", + provider=ModelProvider.ANTHROPIC, + display_name="Claude 3 Opus", + description="Most capable Claude model", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=200000, + ), + ), + AvailableModel( + id="claude-3-haiku", + provider=ModelProvider.ANTHROPIC, + display_name="Claude 3 Haiku", + description="Fastest Claude model", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=200000, + ), + ), +] + + +GOOGLE_MODELS = [ + AvailableModel( + id="gemini-flash-latest", + provider=ModelProvider.GOOGLE, + display_name="Gemini Flash (Latest)", + description="Latest experimental Gemini model", + capabilities=ModelCapabilities( + supports_tools=True, + supports_structured_output=True, + supported_attachment_types=GOOGLE_ATTACHMENTS, + context_window=int(1e6), + ), + ), + AvailableModel( + id="gemini-flash-lite-latest", + provider=ModelProvider.GOOGLE, + display_name="Gemini Flash Lite (Latest)", + description="Fast with huge context window", + capabilities=ModelCapabilities( + supports_tools=True, + supports_structured_output=True, + supported_attachment_types=GOOGLE_ATTACHMENTS, + context_window=int(1e6), + ), + ), + AvailableModel( + id="gemini-2.5-pro", + provider=ModelProvider.GOOGLE, + display_name="Gemini 2.5 Pro", + description="Most capable Gemini model", + capabilities=ModelCapabilities( + supports_tools=True, + supports_structured_output=True, + supported_attachment_types=GOOGLE_ATTACHMENTS, + context_window=2000000, + ), + ), +] + + +GROQ_MODELS = [ + AvailableModel( + id="llama-3.3-70b-versatile", + provider=ModelProvider.GROQ, + display_name="Llama 3.3 70B", + description="Latest Llama model with high-speed inference", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=128000, + ), + ), + AvailableModel( + id="llama-3.1-8b-instant", + provider=ModelProvider.GROQ, + display_name="Llama 3.1 8B", + description="Ultra-fast inference", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=128000, + ), + ), + AvailableModel( + id="mixtral-8x7b-32768", + provider=ModelProvider.GROQ, + display_name="Mixtral 8x7B", + description="High-speed inference with tools", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=32768, + ), + ), +] + + +AZURE_OPENAI_MODELS = [ + AvailableModel( + id="azure-gpt-4", + provider=ModelProvider.AZURE_OPENAI, + display_name="Azure OpenAI GPT-4", + description="Azure-hosted GPT model", + capabilities=ModelCapabilities( + supports_tools=True, + supports_structured_output=True, + supported_attachment_types=OPENAI_ATTACHMENTS, + context_window=8192, + ), + ), +] diff --git a/application/core/model_settings.py b/application/core/model_settings.py new file mode 100644 index 00000000..87325ac3 --- /dev/null +++ b/application/core/model_settings.py @@ -0,0 +1,236 @@ +import logging +from dataclasses import dataclass, field +from enum import Enum +from typing import Dict, List, Optional + +logger = logging.getLogger(__name__) + + +class ModelProvider(str, Enum): + OPENAI = "openai" + AZURE_OPENAI = "azure_openai" + ANTHROPIC = "anthropic" + GROQ = "groq" + GOOGLE = "google" + HUGGINGFACE = "huggingface" + LLAMA_CPP = "llama.cpp" + DOCSGPT = "docsgpt" + PREMAI = "premai" + SAGEMAKER = "sagemaker" + NOVITA = "novita" + + +@dataclass +class ModelCapabilities: + supports_tools: bool = False + supports_structured_output: bool = False + supports_streaming: bool = True + supported_attachment_types: List[str] = field(default_factory=list) + context_window: int = 128000 + input_cost_per_token: Optional[float] = None + output_cost_per_token: Optional[float] = None + + +@dataclass +class AvailableModel: + id: str + provider: ModelProvider + display_name: str + description: str = "" + capabilities: ModelCapabilities = field(default_factory=ModelCapabilities) + enabled: bool = True + base_url: Optional[str] = None + + def to_dict(self) -> Dict: + result = { + "id": self.id, + "provider": self.provider.value, + "display_name": self.display_name, + "description": self.description, + "supported_attachment_types": self.capabilities.supported_attachment_types, + "supports_tools": self.capabilities.supports_tools, + "supports_structured_output": self.capabilities.supports_structured_output, + "supports_streaming": self.capabilities.supports_streaming, + "context_window": self.capabilities.context_window, + "enabled": self.enabled, + } + if self.base_url: + result["base_url"] = self.base_url + return result + + +class ModelRegistry: + _instance = None + _initialized = False + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if not ModelRegistry._initialized: + self.models: Dict[str, AvailableModel] = {} + self.default_model_id: Optional[str] = None + self._load_models() + ModelRegistry._initialized = True + + @classmethod + def get_instance(cls) -> "ModelRegistry": + return cls() + + def _load_models(self): + from application.core.settings import settings + + self.models.clear() + + self._add_docsgpt_models(settings) + if settings.OPENAI_API_KEY or ( + settings.LLM_PROVIDER == "openai" and settings.API_KEY + ): + self._add_openai_models(settings) + if settings.OPENAI_API_BASE or ( + settings.LLM_PROVIDER == "azure_openai" and settings.API_KEY + ): + self._add_azure_openai_models(settings) + if settings.ANTHROPIC_API_KEY or ( + settings.LLM_PROVIDER == "anthropic" and settings.API_KEY + ): + self._add_anthropic_models(settings) + if settings.GOOGLE_API_KEY or ( + settings.LLM_PROVIDER == "google" and settings.API_KEY + ): + self._add_google_models(settings) + if settings.GROQ_API_KEY or ( + settings.LLM_PROVIDER == "groq" and settings.API_KEY + ): + self._add_groq_models(settings) + if settings.HUGGINGFACE_API_KEY or ( + settings.LLM_PROVIDER == "huggingface" and settings.API_KEY + ): + self._add_huggingface_models(settings) + # Default model selection + + if settings.LLM_NAME and settings.LLM_NAME in self.models: + self.default_model_id = settings.LLM_NAME + elif settings.LLM_PROVIDER and settings.API_KEY: + for model_id, model in self.models.items(): + if model.provider.value == settings.LLM_PROVIDER: + self.default_model_id = model_id + break + else: + self.default_model_id = next(iter(self.models.keys())) + logger.info( + f"ModelRegistry loaded {len(self.models)} models, default: {self.default_model_id}" + ) + + def _add_openai_models(self, settings): + from application.core.model_configs import OPENAI_MODELS + + if settings.OPENAI_API_KEY: + for model in OPENAI_MODELS: + self.models[model.id] = model + return + if settings.LLM_PROVIDER == "openai" and settings.LLM_NAME: + for model in OPENAI_MODELS: + if model.id == settings.LLM_NAME: + self.models[model.id] = model + return + for model in OPENAI_MODELS: + self.models[model.id] = model + + def _add_azure_openai_models(self, settings): + from application.core.model_configs import AZURE_OPENAI_MODELS + + if settings.LLM_PROVIDER == "azure_openai" and settings.LLM_NAME: + for model in AZURE_OPENAI_MODELS: + if model.id == settings.LLM_NAME: + self.models[model.id] = model + return + for model in AZURE_OPENAI_MODELS: + self.models[model.id] = model + + def _add_anthropic_models(self, settings): + from application.core.model_configs import ANTHROPIC_MODELS + + if settings.ANTHROPIC_API_KEY: + for model in ANTHROPIC_MODELS: + self.models[model.id] = model + return + if settings.LLM_PROVIDER == "anthropic" and settings.LLM_NAME: + for model in ANTHROPIC_MODELS: + if model.id == settings.LLM_NAME: + self.models[model.id] = model + return + for model in ANTHROPIC_MODELS: + self.models[model.id] = model + + def _add_google_models(self, settings): + from application.core.model_configs import GOOGLE_MODELS + + if settings.GOOGLE_API_KEY: + for model in GOOGLE_MODELS: + self.models[model.id] = model + return + if settings.LLM_PROVIDER == "google" and settings.LLM_NAME: + for model in GOOGLE_MODELS: + if model.id == settings.LLM_NAME: + self.models[model.id] = model + return + for model in GOOGLE_MODELS: + self.models[model.id] = model + + def _add_groq_models(self, settings): + from application.core.model_configs import GROQ_MODELS + + if settings.GROQ_API_KEY: + for model in GROQ_MODELS: + self.models[model.id] = model + return + if settings.LLM_PROVIDER == "groq" and settings.LLM_NAME: + for model in GROQ_MODELS: + if model.id == settings.LLM_NAME: + self.models[model.id] = model + return + for model in GROQ_MODELS: + self.models[model.id] = model + + def _add_docsgpt_models(self, settings): + model_id = "docsgpt-local" + model = AvailableModel( + id=model_id, + provider=ModelProvider.DOCSGPT, + display_name="DocsGPT Model", + description="Local model", + capabilities=ModelCapabilities( + supports_tools=False, + supported_attachment_types=[], + ), + ) + self.models[model_id] = model + + def _add_huggingface_models(self, settings): + model_id = "huggingface-local" + model = AvailableModel( + id=model_id, + provider=ModelProvider.HUGGINGFACE, + display_name="Hugging Face Model", + description="Local Hugging Face model", + capabilities=ModelCapabilities( + supports_tools=False, + supported_attachment_types=[], + ), + ) + self.models[model_id] = model + + def get_model(self, model_id: str) -> Optional[AvailableModel]: + return self.models.get(model_id) + + def get_all_models(self) -> List[AvailableModel]: + return list(self.models.values()) + + def get_enabled_models(self) -> List[AvailableModel]: + return [m for m in self.models.values() if m.enabled] + + def model_exists(self, model_id: str) -> bool: + return model_id in self.models diff --git a/application/core/model_utils.py b/application/core/model_utils.py new file mode 100644 index 00000000..f24dbf47 --- /dev/null +++ b/application/core/model_utils.py @@ -0,0 +1,91 @@ +from typing import Any, Dict, Optional + +from application.core.model_settings import ModelRegistry + + +def get_api_key_for_provider(provider: str) -> Optional[str]: + """Get the appropriate API key for a provider""" + from application.core.settings import settings + + provider_key_map = { + "openai": settings.OPENAI_API_KEY, + "anthropic": settings.ANTHROPIC_API_KEY, + "google": settings.GOOGLE_API_KEY, + "groq": settings.GROQ_API_KEY, + "huggingface": settings.HUGGINGFACE_API_KEY, + "azure_openai": settings.API_KEY, + "docsgpt": None, + "llama.cpp": None, + } + + provider_key = provider_key_map.get(provider) + if provider_key: + return provider_key + return settings.API_KEY + + +def get_all_available_models() -> Dict[str, Dict[str, Any]]: + """Get all available models with metadata for API response""" + registry = ModelRegistry.get_instance() + return {model.id: model.to_dict() for model in registry.get_enabled_models()} + + +def validate_model_id(model_id: str) -> bool: + """Check if a model ID exists in registry""" + registry = ModelRegistry.get_instance() + return registry.model_exists(model_id) + + +def get_model_capabilities(model_id: str) -> Optional[Dict[str, Any]]: + """Get capabilities for a specific model""" + registry = ModelRegistry.get_instance() + model = registry.get_model(model_id) + if model: + return { + "supported_attachment_types": model.capabilities.supported_attachment_types, + "supports_tools": model.capabilities.supports_tools, + "supports_structured_output": model.capabilities.supports_structured_output, + "context_window": model.capabilities.context_window, + } + return None + + +def get_default_model_id() -> str: + """Get the system default model ID""" + registry = ModelRegistry.get_instance() + return registry.default_model_id + + +def get_provider_from_model_id(model_id: str) -> Optional[str]: + """Get the provider name for a given model_id""" + registry = ModelRegistry.get_instance() + model = registry.get_model(model_id) + if model: + return model.provider.value + return None + + +def get_token_limit(model_id: str) -> int: + """ + Get context window (token limit) for a model. + Returns model's context_window or default 128000 if model not found. + """ + from application.core.settings import settings + + registry = ModelRegistry.get_instance() + model = registry.get_model(model_id) + if model: + return model.capabilities.context_window + return settings.DEFAULT_LLM_TOKEN_LIMIT + + +def get_base_url_for_model(model_id: str) -> Optional[str]: + """ + Get the custom base_url for a specific model if configured. + Returns None if no custom base_url is set. + """ + registry = ModelRegistry.get_instance() + model = registry.get_model(model_id) + if model: + return model.base_url + return None diff --git a/application/core/settings.py b/application/core/settings.py index 22116a7c..ee7ffa05 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -22,15 +22,7 @@ class Settings(BaseSettings): MONGO_DB_NAME: str = "docsgpt" LLM_PATH: str = os.path.join(current_dir, "models/docsgpt-7b-f16.gguf") DEFAULT_MAX_HISTORY: int = 150 - LLM_TOKEN_LIMITS: dict = { - "gpt-4o": 128000, - "gpt-4o-mini": 128000, - "gpt-4": 8192, - "gpt-3.5-turbo": 4096, - "claude-2": int(1e5), - "gemini-2.5-flash": int(1e6), - } - DEFAULT_LLM_TOKEN_LIMIT: int = 128000 + DEFAULT_LLM_TOKEN_LIMIT: int = 128000 # Fallback when model not found in registry RESERVED_TOKENS: dict = { "system_prompt": 500, "current_query": 500, @@ -64,14 +56,22 @@ class Settings(BaseSettings): ) # GitHub source - GITHUB_ACCESS_TOKEN: Optional[str] = None # PAT token with read repo access + GITHUB_ACCESS_TOKEN: Optional[str] = None # PAT token with read repo access # LLM Cache CACHE_REDIS_URL: str = "redis://localhost:6379/2" API_URL: str = "http://localhost:7091" # backend url for celery worker - API_KEY: Optional[str] = None # LLM api key + API_KEY: Optional[str] = None # LLM api key (used by LLM_PROVIDER) + + # Provider-specific API keys (for multi-model support) + OPENAI_API_KEY: Optional[str] = None + ANTHROPIC_API_KEY: Optional[str] = None + GOOGLE_API_KEY: Optional[str] = None + GROQ_API_KEY: Optional[str] = None + HUGGINGFACE_API_KEY: Optional[str] = None + EMBEDDINGS_KEY: Optional[str] = ( None # api key for embeddings (if using openai, just copy API_KEY) ) @@ -138,11 +138,12 @@ class Settings(BaseSettings): # Encryption settings ENCRYPTION_SECRET_KEY: str = "default-docsgpt-encryption-key" - TTS_PROVIDER: str = "google_tts" # google_tts or elevenlabs + TTS_PROVIDER: str = "google_tts" # google_tts or elevenlabs ELEVENLABS_API_KEY: Optional[str] = None # Tool pre-fetch settings ENABLE_TOOL_PREFETCH: bool = True + path = Path(__file__).parent.parent.absolute() settings = Settings(_env_file=path.joinpath(".env"), _env_file_encoding="utf-8") diff --git a/application/llm/anthropic.py b/application/llm/anthropic.py index b55dd855..4d26f925 100644 --- a/application/llm/anthropic.py +++ b/application/llm/anthropic.py @@ -1,30 +1,41 @@ -from application.llm.base import BaseLLM +from anthropic import AI_PROMPT, Anthropic, HUMAN_PROMPT + from application.core.settings import settings +from application.llm.base import BaseLLM class AnthropicLLM(BaseLLM): - def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): - from anthropic import Anthropic, HUMAN_PROMPT, AI_PROMPT + def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs): super().__init__(*args, **kwargs) - self.api_key = ( - api_key or settings.ANTHROPIC_API_KEY - ) # If not provided, use a default from settings + self.api_key = api_key or settings.ANTHROPIC_API_KEY or settings.API_KEY self.user_api_key = user_api_key - self.anthropic = Anthropic(api_key=self.api_key) + + # Use custom base_url if provided + if base_url: + self.anthropic = Anthropic(api_key=self.api_key, base_url=base_url) + else: + self.anthropic = Anthropic(api_key=self.api_key) + self.HUMAN_PROMPT = HUMAN_PROMPT self.AI_PROMPT = AI_PROMPT def _raw_gen( - self, baseself, model, messages, stream=False, tools=None, max_tokens=300, **kwargs + self, + baseself, + model, + messages, + stream=False, + tools=None, + max_tokens=300, + **kwargs, ): context = messages[0]["content"] user_question = messages[-1]["content"] prompt = f"### Context \n {context} \n ### Question \n {user_question}" if stream: return self.gen_stream(model, prompt, stream, max_tokens, **kwargs) - completion = self.anthropic.completions.create( model=model, max_tokens_to_sample=max_tokens, @@ -34,7 +45,14 @@ class AnthropicLLM(BaseLLM): return completion.completion def _raw_gen_stream( - self, baseself, model, messages, stream=True, tools=None, max_tokens=300, **kwargs + self, + baseself, + model, + messages, + stream=True, + tools=None, + max_tokens=300, + **kwargs, ): context = messages[0]["content"] user_question = messages[-1]["content"] @@ -50,5 +68,5 @@ class AnthropicLLM(BaseLLM): for completion in stream_response: yield completion.completion finally: - if hasattr(stream_response, 'close'): + if hasattr(stream_response, "close"): stream_response.close() diff --git a/application/llm/base.py b/application/llm/base.py index c16ec99e..e8ee78eb 100644 --- a/application/llm/base.py +++ b/application/llm/base.py @@ -13,30 +13,32 @@ class BaseLLM(ABC): def __init__( self, decoded_token=None, + model_id=None, + base_url=None, ): self.decoded_token = decoded_token + self.model_id = model_id + self.base_url = base_url self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0} - self.fallback_provider = settings.FALLBACK_LLM_PROVIDER - self.fallback_model_name = settings.FALLBACK_LLM_NAME - self.fallback_llm_api_key = settings.FALLBACK_LLM_API_KEY self._fallback_llm = None + self._fallback_sequence_index = 0 @property def fallback_llm(self): - """Lazy-loaded fallback LLM instance.""" - if ( - self._fallback_llm is None - and self.fallback_provider - and self.fallback_model_name - ): + """Lazy-loaded fallback LLM from FALLBACK_* settings.""" + if self._fallback_llm is None and settings.FALLBACK_LLM_PROVIDER: try: from application.llm.llm_creator import LLMCreator self._fallback_llm = LLMCreator.create_llm( - self.fallback_provider, - self.fallback_llm_api_key, - None, - self.decoded_token, + settings.FALLBACK_LLM_PROVIDER, + api_key=settings.FALLBACK_LLM_API_KEY or settings.API_KEY, + user_api_key=None, + decoded_token=self.decoded_token, + model_id=settings.FALLBACK_LLM_NAME, + ) + logger.info( + f"Fallback LLM initialized: {settings.FALLBACK_LLM_PROVIDER}/{settings.FALLBACK_LLM_NAME}" ) except Exception as e: logger.error( @@ -54,7 +56,7 @@ class BaseLLM(ABC): self, method_name: str, decorators: list, *args, **kwargs ): """ - Unified method execution with fallback support. + Execute method with fallback support. Args: method_name: Name of the raw method ('_raw_gen' or '_raw_gen_stream') @@ -73,10 +75,10 @@ class BaseLLM(ABC): return decorated_method() except Exception as e: if not self.fallback_llm: - logger.error(f"Primary LLM failed and no fallback available: {str(e)}") + logger.error(f"Primary LLM failed and no fallback configured: {str(e)}") raise logger.warning( - f"Falling back to {self.fallback_provider}/{self.fallback_model_name}. Error: {str(e)}" + f"Primary LLM failed. Falling back to {settings.FALLBACK_LLM_PROVIDER}/{settings.FALLBACK_LLM_NAME}. Error: {str(e)}" ) fallback_method = getattr( diff --git a/application/llm/docsgpt_provider.py b/application/llm/docsgpt_provider.py index 3572db40..44a479ae 100644 --- a/application/llm/docsgpt_provider.py +++ b/application/llm/docsgpt_provider.py @@ -1,5 +1,7 @@ import json +from openai import OpenAI + from application.core.settings import settings from application.llm.base import BaseLLM @@ -7,12 +9,11 @@ from application.llm.base import BaseLLM class DocsGPTAPILLM(BaseLLM): def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): - from openai import OpenAI super().__init__(*args, **kwargs) - self.client = OpenAI(api_key="sk-docsgpt-public", base_url="https://oai.arc53.com") + self.api_key = "sk-docsgpt-public" + self.client = OpenAI(api_key=self.api_key, base_url="https://oai.arc53.com") self.user_api_key = user_api_key - self.api_key = api_key def _clean_messages_openai(self, messages): cleaned_messages = [] @@ -22,7 +23,6 @@ class DocsGPTAPILLM(BaseLLM): if role == "model": role = "assistant" - if role and content is not None: if isinstance(content, str): cleaned_messages.append({"role": role, "content": content}) @@ -69,7 +69,6 @@ class DocsGPTAPILLM(BaseLLM): ) else: raise ValueError(f"Unexpected content type: {type(content)}") - return cleaned_messages def _raw_gen( @@ -121,7 +120,6 @@ class DocsGPTAPILLM(BaseLLM): response = self.client.chat.completions.create( model="docsgpt", messages=messages, stream=stream, **kwargs ) - try: for line in response: if ( @@ -133,8 +131,8 @@ class DocsGPTAPILLM(BaseLLM): elif len(line.choices) > 0: yield line.choices[0] finally: - if hasattr(response, 'close'): + if hasattr(response, "close"): response.close() def _supports_tools(self): - return True \ No newline at end of file + return True diff --git a/application/llm/google_ai.py b/application/llm/google_ai.py index 47be51cd..9c58a3e1 100644 --- a/application/llm/google_ai.py +++ b/application/llm/google_ai.py @@ -13,8 +13,9 @@ from application.storage.storage_creator import StorageCreator class GoogleLLM(BaseLLM): def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): super().__init__(*args, **kwargs) - self.api_key = api_key + self.api_key = api_key or settings.GOOGLE_API_KEY or settings.API_KEY self.user_api_key = user_api_key + self.client = genai.Client(api_key=self.api_key) self.storage = StorageCreator.get_storage() @@ -47,21 +48,19 @@ class GoogleLLM(BaseLLM): """ if not attachments: return messages - prepared_messages = messages.copy() # Find the user message to attach files to the last one + user_message_index = None for i in range(len(prepared_messages) - 1, -1, -1): if prepared_messages[i].get("role") == "user": user_message_index = i break - if user_message_index is None: user_message = {"role": "user", "content": []} prepared_messages.append(user_message) user_message_index = len(prepared_messages) - 1 - if isinstance(prepared_messages[user_message_index].get("content"), str): text_content = prepared_messages[user_message_index]["content"] prepared_messages[user_message_index]["content"] = [ @@ -69,7 +68,6 @@ class GoogleLLM(BaseLLM): ] elif not isinstance(prepared_messages[user_message_index].get("content"), list): prepared_messages[user_message_index]["content"] = [] - files = [] for attachment in attachments: mime_type = attachment.get("mime_type") @@ -92,11 +90,9 @@ class GoogleLLM(BaseLLM): "text": f"[File could not be processed: {attachment.get('path', 'unknown')}]", } ) - if files: logging.info(f"GoogleLLM: Adding {len(files)} files to message") prepared_messages[user_message_index]["content"].append({"files": files}) - return prepared_messages def _upload_file_to_google(self, attachment): @@ -111,14 +107,11 @@ class GoogleLLM(BaseLLM): """ if "google_file_uri" in attachment: return attachment["google_file_uri"] - file_path = attachment.get("path") if not file_path: raise ValueError("No file path provided in attachment") - if not self.storage.file_exists(file_path): raise FileNotFoundError(f"File not found: {file_path}") - try: file_uri = self.storage.process_file( file_path, @@ -136,7 +129,6 @@ class GoogleLLM(BaseLLM): attachments_collection.update_one( {"_id": attachment["_id"]}, {"$set": {"google_file_uri": file_uri}} ) - return file_uri except Exception as e: logging.error(f"Error uploading file to Google AI: {e}", exc_info=True) @@ -153,7 +145,6 @@ class GoogleLLM(BaseLLM): role = "model" elif role == "tool": role = "model" - parts = [] if role and content is not None: if isinstance(content, str): @@ -164,6 +155,7 @@ class GoogleLLM(BaseLLM): parts.append(types.Part.from_text(text=item["text"])) elif "function_call" in item: # Remove null values from args to avoid API errors + cleaned_args = self._remove_null_values( item["function_call"]["args"] ) @@ -194,10 +186,8 @@ class GoogleLLM(BaseLLM): ) else: raise ValueError(f"Unexpected content type: {type(content)}") - if parts: cleaned_messages.append(types.Content(role=role, parts=parts)) - return cleaned_messages def _clean_schema(self, schema_obj): @@ -233,8 +223,8 @@ class GoogleLLM(BaseLLM): cleaned[key] = [self._clean_schema(item) for item in value] else: cleaned[key] = value - # Validate that required properties actually exist in properties + if "required" in cleaned and "properties" in cleaned: valid_required = [] properties_keys = set(cleaned["properties"].keys()) @@ -247,7 +237,6 @@ class GoogleLLM(BaseLLM): cleaned.pop("required", None) elif "required" in cleaned and "properties" not in cleaned: cleaned.pop("required", None) - return cleaned def _clean_tools_format(self, tools_list): @@ -263,7 +252,6 @@ class GoogleLLM(BaseLLM): cleaned_properties = {} for k, v in properties.items(): cleaned_properties[k] = self._clean_schema(v) - genai_function = dict( name=function["name"], description=function["description"], @@ -282,10 +270,8 @@ class GoogleLLM(BaseLLM): name=function["name"], description=function["description"], ) - genai_tool = types.Tool(function_declarations=[genai_function]) genai_tools.append(genai_tool) - return genai_tools def _raw_gen( @@ -307,16 +293,14 @@ class GoogleLLM(BaseLLM): if messages[0].role == "system": config.system_instruction = messages[0].parts[0].text messages = messages[1:] - if tools: cleaned_tools = self._clean_tools_format(tools) config.tools = cleaned_tools - # Add response schema for structured output if provided + if response_schema: config.response_schema = response_schema config.response_mime_type = "application/json" - response = client.models.generate_content( model=model, contents=messages, @@ -347,17 +331,16 @@ class GoogleLLM(BaseLLM): if messages[0].role == "system": config.system_instruction = messages[0].parts[0].text messages = messages[1:] - if tools: cleaned_tools = self._clean_tools_format(tools) config.tools = cleaned_tools - # Add response schema for structured output if provided + if response_schema: config.response_schema = response_schema config.response_mime_type = "application/json" - # Check if we have both tools and file attachments + has_attachments = False for message in messages: for part in message.parts: @@ -366,7 +349,6 @@ class GoogleLLM(BaseLLM): break if has_attachments: break - logging.info( f"GoogleLLM: Starting stream generation. Model: {model}, Messages: {json.dumps(messages, default=str)}, Has attachments: {has_attachments}" ) @@ -405,7 +387,6 @@ class GoogleLLM(BaseLLM): """Convert JSON schema to Google AI structured output format.""" if not json_schema: return None - type_map = { "object": "OBJECT", "array": "ARRAY", @@ -418,12 +399,10 @@ class GoogleLLM(BaseLLM): def convert(schema): if not isinstance(schema, dict): return schema - result = {} schema_type = schema.get("type") if schema_type: result["type"] = type_map.get(schema_type.lower(), schema_type.upper()) - for key in [ "description", "nullable", @@ -435,7 +414,6 @@ class GoogleLLM(BaseLLM): ]: if key in schema: result[key] = schema[key] - if "format" in schema: format_value = schema["format"] if schema_type == "string": @@ -445,21 +423,17 @@ class GoogleLLM(BaseLLM): result["format"] = format_value else: result["format"] = format_value - if "properties" in schema: result["properties"] = { k: convert(v) for k, v in schema["properties"].items() } if "propertyOrdering" not in result and result.get("type") == "OBJECT": result["propertyOrdering"] = list(result["properties"].keys()) - if "items" in schema: result["items"] = convert(schema["items"]) - for field in ["anyOf", "oneOf", "allOf"]: if field in schema: result[field] = [convert(s) for s in schema[field]] - return result try: diff --git a/application/llm/groq.py b/application/llm/groq.py index 282d7f47..c2ae40ee 100644 --- a/application/llm/groq.py +++ b/application/llm/groq.py @@ -1,13 +1,18 @@ -from application.llm.base import BaseLLM from openai import OpenAI +from application.core.settings import settings +from application.llm.base import BaseLLM + class GroqLLM(BaseLLM): def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): + super().__init__(*args, **kwargs) - self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1") - self.api_key = api_key + self.api_key = api_key or settings.GROQ_API_KEY or settings.API_KEY self.user_api_key = user_api_key + self.client = OpenAI( + api_key=self.api_key, base_url="https://api.groq.com/openai/v1" + ) def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs): if tools: diff --git a/application/llm/handlers/base.py b/application/llm/handlers/base.py index 96ed4c00..920caf65 100644 --- a/application/llm/handlers/base.py +++ b/application/llm/handlers/base.py @@ -282,7 +282,7 @@ class LLMHandler(ABC): messages = e.value break response = agent.llm.gen( - model=agent.gpt_model, messages=messages, tools=agent.tools + model=agent.model_id, messages=messages, tools=agent.tools ) parsed = self.parse_response(response) self.llm_calls.append(build_stack_data(agent.llm)) @@ -337,7 +337,7 @@ class LLMHandler(ABC): tool_calls = {} response = agent.llm.gen_stream( - model=agent.gpt_model, messages=messages, tools=agent.tools + model=agent.model_id, messages=messages, tools=agent.tools ) self.llm_calls.append(build_stack_data(agent.llm)) diff --git a/application/llm/llm_creator.py b/application/llm/llm_creator.py index 3ed23854..21d653b9 100644 --- a/application/llm/llm_creator.py +++ b/application/llm/llm_creator.py @@ -1,13 +1,17 @@ -from application.llm.groq import GroqLLM -from application.llm.openai import OpenAILLM, AzureOpenAILLM -from application.llm.sagemaker import SagemakerAPILLM -from application.llm.huggingface import HuggingFaceLLM -from application.llm.llama_cpp import LlamaCpp +import logging + from application.llm.anthropic import AnthropicLLM from application.llm.docsgpt_provider import DocsGPTAPILLM -from application.llm.premai import PremAILLM from application.llm.google_ai import GoogleLLM +from application.llm.groq import GroqLLM +from application.llm.huggingface import HuggingFaceLLM +from application.llm.llama_cpp import LlamaCpp from application.llm.novita import NovitaLLM +from application.llm.openai import AzureOpenAILLM, OpenAILLM +from application.llm.premai import PremAILLM +from application.llm.sagemaker import SagemakerAPILLM + +logger = logging.getLogger(__name__) class LLMCreator: @@ -26,10 +30,26 @@ class LLMCreator: } @classmethod - def create_llm(cls, type, api_key, user_api_key, decoded_token, *args, **kwargs): + def create_llm( + cls, type, api_key, user_api_key, decoded_token, model_id=None, *args, **kwargs + ): + from application.core.model_utils import get_base_url_for_model + llm_class = cls.llms.get(type.lower()) if not llm_class: raise ValueError(f"No LLM class found for type {type}") + + # Extract base_url from model configuration if model_id is provided + base_url = None + if model_id: + base_url = get_base_url_for_model(model_id) + return llm_class( - api_key, user_api_key, decoded_token=decoded_token, *args, **kwargs + api_key, + user_api_key, + decoded_token=decoded_token, + model_id=model_id, + base_url=base_url, + *args, + **kwargs, ) diff --git a/application/llm/openai.py b/application/llm/openai.py index de24e2c5..beab465b 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -2,6 +2,8 @@ import base64 import json import logging +from openai import OpenAI + from application.core.settings import settings from application.llm.base import BaseLLM from application.storage.storage_creator import StorageCreator @@ -9,20 +11,25 @@ from application.storage.storage_creator import StorageCreator class OpenAILLM(BaseLLM): - def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): - from openai import OpenAI + def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs): super().__init__(*args, **kwargs) - if ( + self.api_key = api_key or settings.OPENAI_API_KEY or settings.API_KEY + self.user_api_key = user_api_key + + # Priority: 1) Parameter base_url, 2) Settings OPENAI_BASE_URL, 3) Default + effective_base_url = None + if base_url and isinstance(base_url, str) and base_url.strip(): + effective_base_url = base_url + elif ( isinstance(settings.OPENAI_BASE_URL, str) and settings.OPENAI_BASE_URL.strip() ): - self.client = OpenAI(api_key=api_key, base_url=settings.OPENAI_BASE_URL) + effective_base_url = settings.OPENAI_BASE_URL else: - DEFAULT_OPENAI_API_BASE = "https://api.openai.com/v1" - self.client = OpenAI(api_key=api_key, base_url=DEFAULT_OPENAI_API_BASE) - self.api_key = api_key - self.user_api_key = user_api_key + effective_base_url = "https://api.openai.com/v1" + + self.client = OpenAI(api_key=self.api_key, base_url=effective_base_url) self.storage = StorageCreator.get_storage() def _clean_messages_openai(self, messages): @@ -33,7 +40,6 @@ class OpenAILLM(BaseLLM): if role == "model": role = "assistant" - if role and content is not None: if isinstance(content, str): cleaned_messages.append({"role": role, "content": content}) @@ -107,7 +113,6 @@ class OpenAILLM(BaseLLM): ) else: raise ValueError(f"Unexpected content type: {type(content)}") - return cleaned_messages def _raw_gen( @@ -132,10 +137,8 @@ class OpenAILLM(BaseLLM): if tools: request_params["tools"] = tools - if response_format: request_params["response_format"] = response_format - response = self.client.chat.completions.create(**request_params) if tools: @@ -165,10 +168,8 @@ class OpenAILLM(BaseLLM): if tools: request_params["tools"] = tools - if response_format: request_params["response_format"] = response_format - response = self.client.chat.completions.create(**request_params) try: @@ -194,7 +195,6 @@ class OpenAILLM(BaseLLM): def prepare_structured_output_format(self, json_schema): if not json_schema: return None - try: def add_additional_properties_false(schema_obj): @@ -204,11 +204,11 @@ class OpenAILLM(BaseLLM): if schema_copy.get("type") == "object": schema_copy["additionalProperties"] = False # Ensure 'required' includes all properties for OpenAI strict mode + if "properties" in schema_copy: schema_copy["required"] = list( schema_copy["properties"].keys() ) - for key, value in schema_copy.items(): if key == "properties" and isinstance(value, dict): schema_copy[key] = { @@ -224,7 +224,6 @@ class OpenAILLM(BaseLLM): add_additional_properties_false(sub_schema) for sub_schema in value ] - return schema_copy return schema_obj @@ -243,7 +242,6 @@ class OpenAILLM(BaseLLM): } return result - except Exception as e: logging.error(f"Error preparing structured output format: {e}") return None @@ -277,21 +275,19 @@ class OpenAILLM(BaseLLM): """ if not attachments: return messages - prepared_messages = messages.copy() # Find the user message to attach file_id to the last one + user_message_index = None for i in range(len(prepared_messages) - 1, -1, -1): if prepared_messages[i].get("role") == "user": user_message_index = i break - if user_message_index is None: user_message = {"role": "user", "content": []} prepared_messages.append(user_message) user_message_index = len(prepared_messages) - 1 - if isinstance(prepared_messages[user_message_index].get("content"), str): text_content = prepared_messages[user_message_index]["content"] prepared_messages[user_message_index]["content"] = [ @@ -299,7 +295,6 @@ class OpenAILLM(BaseLLM): ] elif not isinstance(prepared_messages[user_message_index].get("content"), list): prepared_messages[user_message_index]["content"] = [] - for attachment in attachments: mime_type = attachment.get("mime_type") @@ -326,6 +321,7 @@ class OpenAILLM(BaseLLM): } ) # Handle PDFs using the file API + elif mime_type == "application/pdf": try: file_id = self._upload_file_to_openai(attachment) @@ -341,7 +337,6 @@ class OpenAILLM(BaseLLM): "text": f"File content:\n\n{attachment['content']}", } ) - return prepared_messages def _get_base64_image(self, attachment): @@ -357,7 +352,6 @@ class OpenAILLM(BaseLLM): file_path = attachment.get("path") if not file_path: raise ValueError("No file path provided in attachment") - try: with self.storage.get_file(file_path) as image_file: return base64.b64encode(image_file.read()).decode("utf-8") @@ -381,12 +375,10 @@ class OpenAILLM(BaseLLM): if "openai_file_id" in attachment: return attachment["openai_file_id"] - file_path = attachment.get("path") if not self.storage.file_exists(file_path): raise FileNotFoundError(f"File not found: {file_path}") - try: file_id = self.storage.process_file( file_path, @@ -404,7 +396,6 @@ class OpenAILLM(BaseLLM): attachments_collection.update_one( {"_id": attachment["_id"]}, {"$set": {"openai_file_id": file_id}} ) - return file_id except Exception as e: logging.error(f"Error uploading file to OpenAI: {e}", exc_info=True) diff --git a/application/retriever/classic_rag.py b/application/retriever/classic_rag.py index f0428a26..f0468a18 100644 --- a/application/retriever/classic_rag.py +++ b/application/retriever/classic_rag.py @@ -16,7 +16,7 @@ class ClassicRAG(BaseRetriever): prompt="", chunks=2, doc_token_limit=50000, - gpt_model="docsgpt", + model_id="docsgpt-local", user_api_key=None, llm_name=settings.LLM_PROVIDER, api_key=settings.API_KEY, @@ -40,7 +40,7 @@ class ClassicRAG(BaseRetriever): f"ClassicRAG initialized with chunks={self.chunks}, user_api_key={user_identifier}, " f"sources={'active_docs' in source and source['active_docs'] is not None}" ) - self.gpt_model = gpt_model + self.model_id = model_id self.doc_token_limit = doc_token_limit self.user_api_key = user_api_key self.llm_name = llm_name @@ -100,7 +100,7 @@ class ClassicRAG(BaseRetriever): ] try: - rephrased_query = self.llm.gen(model=self.gpt_model, messages=messages) + rephrased_query = self.llm.gen(model=self.model_id, messages=messages) print(f"Rephrased query: {rephrased_query}") return rephrased_query if rephrased_query else self.original_question except Exception as e: diff --git a/application/utils.py b/application/utils.py index 06eaf495..89b884f0 100644 --- a/application/utils.py +++ b/application/utils.py @@ -7,6 +7,8 @@ import tiktoken from flask import jsonify, make_response from werkzeug.utils import secure_filename +from application.core.model_utils import get_token_limit + from application.core.settings import settings @@ -75,11 +77,9 @@ def count_tokens_docs(docs): def calculate_doc_token_budget( - gpt_model: str = "gpt-4o", history_token_limit: int = 2000 + model_id: str = "gpt-4o", history_token_limit: int = 2000 ) -> int: - total_context = settings.LLM_TOKEN_LIMITS.get( - gpt_model, settings.DEFAULT_LLM_TOKEN_LIMIT - ) + total_context = get_token_limit(model_id) reserved = sum(settings.RESERVED_TOKENS.values()) doc_budget = total_context - history_token_limit - reserved return max(doc_budget, 1000) @@ -144,16 +144,13 @@ def get_hash(data): return hashlib.md5(data.encode(), usedforsecurity=False).hexdigest() -def limit_chat_history(history, max_token_limit=None, gpt_model="docsgpt"): +def limit_chat_history(history, max_token_limit=None, model_id="docsgpt-local"): """Limit chat history to fit within token limit.""" - from application.core.settings import settings - + model_token_limit = get_token_limit(model_id) max_token_limit = ( max_token_limit - if max_token_limit - and max_token_limit - < settings.LLM_TOKEN_LIMITS.get(gpt_model, settings.DEFAULT_LLM_TOKEN_LIMIT) - else settings.LLM_TOKEN_LIMITS.get(gpt_model, settings.DEFAULT_LLM_TOKEN_LIMIT) + if max_token_limit and max_token_limit < model_token_limit + else model_token_limit ) if not history: @@ -205,37 +202,44 @@ def clean_text_for_tts(text: str) -> str: clean text for Text-to-Speech processing. """ # Handle code blocks and links - text = re.sub(r'```mermaid[\s\S]*?```', ' flowchart, ', text) ## ```mermaid...``` - text = re.sub(r'```[\s\S]*?```', ' code block, ', text) ## ```code``` - text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text) ## [text](url) - text = re.sub(r'!\[([^\]]*)\]\([^\)]+\)', '', text) ## ![alt](url) + + text = re.sub(r"```mermaid[\s\S]*?```", " flowchart, ", text) ## ```mermaid...``` + text = re.sub(r"```[\s\S]*?```", " code block, ", text) ## ```code``` + text = re.sub(r"\[([^\]]+)\]\([^\)]+\)", r"\1", text) ## [text](url) + text = re.sub(r"!\[([^\]]*)\]\([^\)]+\)", "", text) ## ![alt](url) # Remove markdown formatting - text = re.sub(r'`([^`]+)`', r'\1', text) ## `code` - text = re.sub(r'\{([^}]*)\}', r' \1 ', text) ## {text} - text = re.sub(r'[{}]', ' ', text) ## unmatched {} - text = re.sub(r'\[([^\]]+)\]', r' \1 ', text) ## [text] - text = re.sub(r'[\[\]]', ' ', text) ## unmatched [] - text = re.sub(r'(\*\*|__)(.*?)\1', r'\2', text) ## **bold** __bold__ - text = re.sub(r'(\*|_)(.*?)\1', r'\2', text) ## *italic* _italic_ - text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE) ## # headers - text = re.sub(r'^>\s+', '', text, flags=re.MULTILINE) ## > blockquotes - text = re.sub(r'^[\s]*[-\*\+]\s+', '', text, flags=re.MULTILINE) ## - * + lists - text = re.sub(r'^[\s]*\d+\.\s+', '', text, flags=re.MULTILINE) ## 1. numbered lists - text = re.sub(r'^[\*\-_]{3,}\s*$', '', text, flags=re.MULTILINE) ## --- *** ___ rules - text = re.sub(r'<[^>]*>', '', text) ## tags - #Remove non-ASCII (emojis, special Unicode) - text = re.sub(r'[^\x20-\x7E\n\r\t]', '', text) + text = re.sub(r"`([^`]+)`", r"\1", text) ## `code` + text = re.sub(r"\{([^}]*)\}", r" \1 ", text) ## {text} + text = re.sub(r"[{}]", " ", text) ## unmatched {} + text = re.sub(r"\[([^\]]+)\]", r" \1 ", text) ## [text] + text = re.sub(r"[\[\]]", " ", text) ## unmatched [] + text = re.sub(r"(\*\*|__)(.*?)\1", r"\2", text) ## **bold** __bold__ + text = re.sub(r"(\*|_)(.*?)\1", r"\2", text) ## *italic* _italic_ + text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE) ## # headers + text = re.sub(r"^>\s+", "", text, flags=re.MULTILINE) ## > blockquotes + text = re.sub(r"^[\s]*[-\*\+]\s+", "", text, flags=re.MULTILINE) ## - * + lists + text = re.sub(r"^[\s]*\d+\.\s+", "", text, flags=re.MULTILINE) ## 1. numbered lists + text = re.sub( + r"^[\*\-_]{3,}\s*$", "", text, flags=re.MULTILINE + ) ## --- *** ___ rules + text = re.sub(r"<[^>]*>", "", text) ## tags - #Replace special sequences - text = re.sub(r'-->', ', ', text) ## --> - text = re.sub(r'<--', ', ', text) ## <-- - text = re.sub(r'=>', ', ', text) ## => - text = re.sub(r'::', ' ', text) ## :: + # Remove non-ASCII (emojis, special Unicode) - #Normalize whitespace - text = re.sub(r'\s+', ' ', text) + text = re.sub(r"[^\x20-\x7E\n\r\t]", "", text) + + # Replace special sequences + + text = re.sub(r"-->", ", ", text) ## --> + text = re.sub(r"<--", ", ", text) ## <-- + text = re.sub(r"=>", ", ", text) ## => + text = re.sub(r"::", " ", text) ## :: + + # Normalize whitespace + + text = re.sub(r"\s+", " ", text) text = text.strip() return text diff --git a/application/worker.py b/application/worker.py index f17e1537..3f957527 100755 --- a/application/worker.py +++ b/application/worker.py @@ -165,7 +165,7 @@ def run_agent_logic(agent_config, input_data): agent_type, endpoint="webhook", llm_name=settings.LLM_PROVIDER, - gpt_model=settings.LLM_NAME, + model_id=settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=user_api_key, prompt=prompt, @@ -180,7 +180,7 @@ def run_agent_logic(agent_config, input_data): prompt=prompt, chunks=chunks, token_limit=settings.DEFAULT_MAX_HISTORY, - gpt_model=settings.LLM_NAME, + model_id=settings.LLM_NAME, user_api_key=user_api_key, decoded_token=decoded_token, ) diff --git a/frontend/src/Hero.tsx b/frontend/src/Hero.tsx index 9b17c10f..8b08430b 100644 --- a/frontend/src/Hero.tsx +++ b/frontend/src/Hero.tsx @@ -1,6 +1,8 @@ -import DocsGPT3 from './assets/cute_docsgpt3.svg'; import { useTranslation } from 'react-i18next'; +import DocsGPT3 from './assets/cute_docsgpt3.svg'; +import DropdownModel from './components/DropdownModel'; + export default function Hero({ handleQuestion, }: { @@ -26,6 +28,10 @@ export default function Hero({ DocsGPT docsgpt + {/* Model Selector */} +
+ +
{/* Demo Buttons Section */} @@ -38,7 +44,7 @@ export default function Hero({ + setIsModelsPopupOpen(false)} + anchorRef={modelAnchorButtonRef} + options={availableModels.map((model) => ({ + id: model.id, + label: model.display_name, + }))} + selectedIds={selectedModelIds} + onSelectionChange={(newSelectedIds: Set) => + setSelectedModelIds( + new Set(Array.from(newSelectedIds).map(String)), + ) + } + title={t('agents.form.modelsPopup.title')} + searchPlaceholder={t( + 'agents.form.modelsPopup.searchPlaceholder', + )} + noOptionsMessage={t('agents.form.modelsPopup.noOptionsMessage')} + /> + {selectedModelIds.size > 0 && ( +
+ + selectedModelIds.has(m.id)) + .map((m) => ({ + label: m.display_name, + value: m.id, + }))} + selectedValue={ + availableModels.find( + (m) => m.id === agent.default_model_id, + )?.display_name || null + } + onSelect={(option: { label: string; value: string }) => + setAgent({ ...agent, default_model_id: option.value }) + } + size="w-full" + rounded="3xl" + border="border" + buttonClassName="bg-white dark:bg-[#222327] border-silver dark:border-[#7E7E7E]" + optionsClassName="bg-white dark:bg-[#383838] border-silver dark:border-[#7E7E7E]" + placeholder={t( + 'agents.form.placeholders.selectDefaultModel', + )} + placeholderClassName="text-gray-400 dark:text-silver" + contentSize="text-sm" + /> +
+ )} + +
-
-
-🎃 Hacktoberfest Prizes, Rules & Q&A 🎃 -
-
-
-

From 17698ce77405b8d72fbef36d94369dc0ad90d59e Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 24 Nov 2025 10:44:19 +0000 Subject: [PATCH 04/93] feat: context compression (#2173) * feat: context compression * fix: ruff --- .gitignore | 1 + application/agents/base.py | 69 + application/api/answer/routes/base.py | 39 + .../answer/services/compression/__init__.py | 20 + .../services/compression/message_builder.py | 234 +++ .../services/compression/orchestrator.py | 232 +++ .../services/compression/prompt_builder.py | 149 ++ .../answer/services/compression/service.py | 306 ++++ .../services/compression/threshold_checker.py | 103 ++ .../services/compression/token_counter.py | 103 ++ .../api/answer/services/compression/types.py | 83 ++ .../answer/services/conversation_service.py | 100 ++ .../api/answer/services/stream_processor.py | 80 +- application/core/model_configs.py | 58 +- application/core/settings.py | 7 + application/llm/google_ai.py | 146 +- application/llm/handlers/base.py | 548 ++++++- application/llm/handlers/google.py | 29 +- application/llm/openai.py | 8 + application/prompts/compression/v1.0.txt | 35 + application/requirements.txt | 2 +- application/utils.py | 18 + tests/llm/test_google_llm.py | 2 +- tests/test_agent_token_tracking.py | 325 +++++ tests/test_compression_service.py | 1082 ++++++++++++++ tests/test_integration.py | 1287 +++++++++++++++++ tests/test_model_validation.py | 106 ++ tests/test_token_management.py | 314 ++++ 28 files changed, 5393 insertions(+), 93 deletions(-) create mode 100644 application/api/answer/services/compression/__init__.py create mode 100644 application/api/answer/services/compression/message_builder.py create mode 100644 application/api/answer/services/compression/orchestrator.py create mode 100644 application/api/answer/services/compression/prompt_builder.py create mode 100644 application/api/answer/services/compression/service.py create mode 100644 application/api/answer/services/compression/threshold_checker.py create mode 100644 application/api/answer/services/compression/token_counter.py create mode 100644 application/api/answer/services/compression/types.py create mode 100644 application/prompts/compression/v1.0.txt create mode 100644 tests/test_agent_token_tracking.py create mode 100644 tests/test_compression_service.py create mode 100755 tests/test_integration.py create mode 100644 tests/test_model_validation.py create mode 100644 tests/test_token_management.py diff --git a/.gitignore b/.gitignore index 91abeca1..e0e6280a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ *.py[cod] *$py.class +experiments/ experiments # C extensions diff --git a/application/agents/base.py b/application/agents/base.py index dbf15a1f..b2d79c03 100644 --- a/application/agents/base.py +++ b/application/agents/base.py @@ -34,6 +34,7 @@ class BaseAgent(ABC): token_limit: Optional[int] = settings.DEFAULT_AGENT_LIMITS["token_limit"], limited_request_mode: Optional[bool] = False, request_limit: Optional[int] = settings.DEFAULT_AGENT_LIMITS["request_limit"], + compressed_summary: Optional[str] = None, ): self.endpoint = endpoint self.llm_name = llm_name @@ -64,6 +65,9 @@ class BaseAgent(ABC): self.token_limit = token_limit self.limited_request_mode = limited_request_mode self.request_limit = request_limit + self.compressed_summary = compressed_summary + self.current_token_count = 0 + self.context_limit_reached = False @log_activity() def gen( @@ -276,12 +280,77 @@ class BaseAgent(ABC): for tool_call in self.tool_calls ] + def _calculate_current_context_tokens(self, messages: List[Dict]) -> int: + """ + Calculate total tokens in current context (messages). + + Args: + messages: List of message dicts + + Returns: + Total token count + """ + from application.api.answer.services.compression.token_counter import ( + TokenCounter, + ) + + return TokenCounter.count_message_tokens(messages) + + def _check_context_limit(self, messages: List[Dict]) -> bool: + """ + Check if we're approaching context limit (80%). + + Args: + messages: Current message list + + Returns: + True if at or above 80% of context limit + """ + from application.core.model_utils import get_token_limit + from application.core.settings import settings + + try: + # Calculate current tokens + current_tokens = self._calculate_current_context_tokens(messages) + self.current_token_count = current_tokens + + # Get context limit for model + context_limit = get_token_limit(self.model_id) + + # Calculate threshold (80%) + threshold = int(context_limit * settings.COMPRESSION_THRESHOLD_PERCENTAGE) + + # Check if we've reached the limit + if current_tokens >= threshold: + logger.warning( + f"Context limit approaching: {current_tokens}/{context_limit} tokens " + f"({(current_tokens/context_limit)*100:.1f}%)" + ) + return True + + return False + + except Exception as e: + logger.error(f"Error checking context limit: {str(e)}", exc_info=True) + return False + def _build_messages( self, system_prompt: str, query: str, ) -> List[Dict]: """Build messages using pre-rendered system prompt""" + # Append compression summary to system prompt if present + if self.compressed_summary: + compression_context = ( + "\n\n---\n\n" + "This session is being continued from a previous conversation that " + "has been compressed to fit within context limits. " + "The conversation is summarized below:\n\n" + f"{self.compressed_summary}" + ) + system_prompt = system_prompt + compression_context + messages = [{"role": "system", "content": system_prompt}] for i in self.chat_history: diff --git a/application/api/answer/routes/base.py b/application/api/answer/routes/base.py index aefb66c6..be112729 100644 --- a/application/api/answer/routes/base.py +++ b/application/api/answer/routes/base.py @@ -266,6 +266,26 @@ class BaseAnswerResource: shared_token=shared_token, attachment_ids=attachment_ids, ) + # Persist compression metadata/summary if it exists and wasn't saved mid-execution + compression_meta = getattr(agent, "compression_metadata", None) + compression_saved = getattr(agent, "compression_saved", False) + if conversation_id and compression_meta and not compression_saved: + try: + self.conversation_service.update_compression_metadata( + conversation_id, compression_meta + ) + self.conversation_service.append_compression_message( + conversation_id, compression_meta + ) + agent.compression_saved = True + logger.info( + f"Persisted compression metadata for conversation {conversation_id}" + ) + except Exception as e: + logger.error( + f"Failed to persist compression metadata: {str(e)}", + exc_info=True, + ) else: conversation_id = None id_data = {"type": "id", "id": str(conversation_id)} @@ -328,6 +348,25 @@ class BaseAnswerResource: shared_token=shared_token, attachment_ids=attachment_ids, ) + compression_meta = getattr(agent, "compression_metadata", None) + compression_saved = getattr(agent, "compression_saved", False) + if conversation_id and compression_meta and not compression_saved: + try: + self.conversation_service.update_compression_metadata( + conversation_id, compression_meta + ) + self.conversation_service.append_compression_message( + conversation_id, compression_meta + ) + agent.compression_saved = True + logger.info( + f"Persisted compression metadata for conversation {conversation_id} (partial stream)" + ) + except Exception as e: + logger.error( + f"Failed to persist compression metadata (partial stream): {str(e)}", + exc_info=True, + ) except Exception as e: logger.error( f"Error saving partial response: {str(e)}", exc_info=True diff --git a/application/api/answer/services/compression/__init__.py b/application/api/answer/services/compression/__init__.py new file mode 100644 index 00000000..4cbdb910 --- /dev/null +++ b/application/api/answer/services/compression/__init__.py @@ -0,0 +1,20 @@ +""" +Compression module for managing conversation context compression. + +""" + +from application.api.answer.services.compression.orchestrator import ( + CompressionOrchestrator, +) +from application.api.answer.services.compression.service import CompressionService +from application.api.answer.services.compression.types import ( + CompressionResult, + CompressionMetadata, +) + +__all__ = [ + "CompressionOrchestrator", + "CompressionService", + "CompressionResult", + "CompressionMetadata", +] diff --git a/application/api/answer/services/compression/message_builder.py b/application/api/answer/services/compression/message_builder.py new file mode 100644 index 00000000..93772fe5 --- /dev/null +++ b/application/api/answer/services/compression/message_builder.py @@ -0,0 +1,234 @@ +"""Message reconstruction utilities for compression.""" + +import logging +import uuid +from typing import Dict, List, Optional + +logger = logging.getLogger(__name__) + + +class MessageBuilder: + """Builds message arrays from compressed context.""" + + @staticmethod + def build_from_compressed_context( + system_prompt: str, + compressed_summary: Optional[str], + recent_queries: List[Dict], + include_tool_calls: bool = False, + context_type: str = "pre_request", + ) -> List[Dict]: + """ + Build messages from compressed context. + + Args: + system_prompt: Original system prompt + compressed_summary: Compressed summary (if any) + recent_queries: Recent uncompressed queries + include_tool_calls: Whether to include tool calls from history + context_type: Type of context ('pre_request' or 'mid_execution') + + Returns: + List of message dicts ready for LLM + """ + # Append compression summary to system prompt if present + if compressed_summary: + system_prompt = MessageBuilder._append_compression_context( + system_prompt, compressed_summary, context_type + ) + + messages = [{"role": "system", "content": system_prompt}] + + # Add recent history + for query in recent_queries: + if "prompt" in query and "response" in query: + messages.append({"role": "user", "content": query["prompt"]}) + messages.append({"role": "assistant", "content": query["response"]}) + + # Add tool calls from history if present + if include_tool_calls and "tool_calls" in query: + for tool_call in query["tool_calls"]: + call_id = tool_call.get("call_id") or str(uuid.uuid4()) + + function_call_dict = { + "function_call": { + "name": tool_call.get("action_name"), + "args": tool_call.get("arguments"), + "call_id": call_id, + } + } + function_response_dict = { + "function_response": { + "name": tool_call.get("action_name"), + "response": {"result": tool_call.get("result")}, + "call_id": call_id, + } + } + + messages.append( + {"role": "assistant", "content": [function_call_dict]} + ) + messages.append( + {"role": "tool", "content": [function_response_dict]} + ) + + # If no recent queries (everything was compressed), add a continuation user message + if len(recent_queries) == 0 and compressed_summary: + messages.append({ + "role": "user", + "content": "Please continue with the remaining tasks based on the context above." + }) + logger.info("Added continuation user message to maintain proper turn-taking after full compression") + + return messages + + @staticmethod + def _append_compression_context( + system_prompt: str, compressed_summary: str, context_type: str = "pre_request" + ) -> str: + """ + Append compression context to system prompt. + + Args: + system_prompt: Original system prompt + compressed_summary: Summary to append + context_type: Type of compression context + + Returns: + Updated system prompt + """ + # Remove existing compression context if present + if "This session is being continued" in system_prompt or "Context window limit reached" in system_prompt: + parts = system_prompt.split("\n\n---\n\n") + system_prompt = parts[0] + + # Build appropriate context message based on type + if context_type == "mid_execution": + context_message = ( + "\n\n---\n\n" + "Context window limit reached during execution. " + "Previous conversation has been compressed to fit within limits. " + "The conversation is summarized below:\n\n" + f"{compressed_summary}" + ) + else: # pre_request + context_message = ( + "\n\n---\n\n" + "This session is being continued from a previous conversation that " + "has been compressed to fit within context limits. " + "The conversation is summarized below:\n\n" + f"{compressed_summary}" + ) + + return system_prompt + context_message + + @staticmethod + def rebuild_messages_after_compression( + messages: List[Dict], + compressed_summary: Optional[str], + recent_queries: List[Dict], + include_current_execution: bool = False, + include_tool_calls: bool = False, + ) -> Optional[List[Dict]]: + """ + Rebuild the message list after compression so tool execution can continue. + + Args: + messages: Original message list + compressed_summary: Compressed summary + recent_queries: Recent uncompressed queries + include_current_execution: Whether to preserve current execution messages + include_tool_calls: Whether to include tool calls from history + + Returns: + Rebuilt message list or None if failed + """ + # Find the system message + system_message = next( + (msg for msg in messages if msg.get("role") == "system"), None + ) + if not system_message: + logger.warning("No system message found in messages list") + return None + + # Update system message with compressed summary + if compressed_summary: + content = system_message.get("content", "") + system_message["content"] = MessageBuilder._append_compression_context( + content, compressed_summary, "mid_execution" + ) + logger.info( + "Appended compression summary to system prompt (truncated): %s", + ( + compressed_summary[:500] + "..." + if len(compressed_summary) > 500 + else compressed_summary + ), + ) + + rebuilt_messages = [system_message] + + # Add recent history from compressed context + for query in recent_queries: + if "prompt" in query and "response" in query: + rebuilt_messages.append({"role": "user", "content": query["prompt"]}) + rebuilt_messages.append( + {"role": "assistant", "content": query["response"]} + ) + + # Add tool calls from history if present + if include_tool_calls and "tool_calls" in query: + for tool_call in query["tool_calls"]: + call_id = tool_call.get("call_id") or str(uuid.uuid4()) + + function_call_dict = { + "function_call": { + "name": tool_call.get("action_name"), + "args": tool_call.get("arguments"), + "call_id": call_id, + } + } + function_response_dict = { + "function_response": { + "name": tool_call.get("action_name"), + "response": {"result": tool_call.get("result")}, + "call_id": call_id, + } + } + + rebuilt_messages.append( + {"role": "assistant", "content": [function_call_dict]} + ) + rebuilt_messages.append( + {"role": "tool", "content": [function_response_dict]} + ) + + # If no recent queries (everything was compressed), add a continuation user message + if len(recent_queries) == 0 and compressed_summary: + rebuilt_messages.append({ + "role": "user", + "content": "Please continue with the remaining tasks based on the context above." + }) + logger.info("Added continuation user message to maintain proper turn-taking after full compression") + + if include_current_execution: + # Preserve any messages that were added during the current execution cycle + recent_msg_count = 1 # system message + for query in recent_queries: + if "prompt" in query and "response" in query: + recent_msg_count += 2 + if "tool_calls" in query: + recent_msg_count += len(query["tool_calls"]) * 2 + + if len(messages) > recent_msg_count: + current_execution_messages = messages[recent_msg_count:] + rebuilt_messages.extend(current_execution_messages) + logger.info( + f"Preserved {len(current_execution_messages)} messages from current execution cycle" + ) + + logger.info( + f"Messages rebuilt: {len(messages)} → {len(rebuilt_messages)} messages. " + f"Ready to continue tool execution." + ) + return rebuilt_messages diff --git a/application/api/answer/services/compression/orchestrator.py b/application/api/answer/services/compression/orchestrator.py new file mode 100644 index 00000000..797a66d4 --- /dev/null +++ b/application/api/answer/services/compression/orchestrator.py @@ -0,0 +1,232 @@ +"""High-level compression orchestration.""" + +import logging +from typing import Any, Dict, Optional + +from application.api.answer.services.compression.service import CompressionService +from application.api.answer.services.compression.threshold_checker import ( + CompressionThresholdChecker, +) +from application.api.answer.services.compression.types import CompressionResult +from application.api.answer.services.conversation_service import ConversationService +from application.core.model_utils import ( + get_api_key_for_provider, + get_provider_from_model_id, +) +from application.core.settings import settings +from application.llm.llm_creator import LLMCreator + +logger = logging.getLogger(__name__) + + +class CompressionOrchestrator: + """ + Facade for compression operations. + + Coordinates between all compression components and provides + a simple interface for callers. + """ + + def __init__( + self, + conversation_service: ConversationService, + threshold_checker: Optional[CompressionThresholdChecker] = None, + ): + """ + Initialize orchestrator. + + Args: + conversation_service: Service for DB operations + threshold_checker: Custom threshold checker (optional) + """ + self.conversation_service = conversation_service + self.threshold_checker = threshold_checker or CompressionThresholdChecker() + + def compress_if_needed( + self, + conversation_id: str, + user_id: str, + model_id: str, + decoded_token: Dict[str, Any], + current_query_tokens: int = 500, + ) -> CompressionResult: + """ + Check if compression is needed and perform it if so. + + This is the main entry point for compression operations. + + Args: + conversation_id: Conversation ID + user_id: User ID + model_id: Model being used for conversation + decoded_token: User's decoded JWT token + current_query_tokens: Estimated tokens for current query + + Returns: + CompressionResult with summary and recent queries + """ + try: + # Load conversation + conversation = self.conversation_service.get_conversation( + conversation_id, user_id + ) + + if not conversation: + logger.warning( + f"Conversation {conversation_id} not found for user {user_id}" + ) + return CompressionResult.failure("Conversation not found") + + # Check if compression is needed + if not self.threshold_checker.should_compress( + conversation, model_id, current_query_tokens + ): + # No compression needed, return full history + queries = conversation.get("queries", []) + return CompressionResult.success_no_compression(queries) + + # Perform compression + return self._perform_compression( + conversation_id, conversation, model_id, decoded_token + ) + + except Exception as e: + logger.error( + f"Error in compress_if_needed: {str(e)}", exc_info=True + ) + return CompressionResult.failure(str(e)) + + def _perform_compression( + self, + conversation_id: str, + conversation: Dict[str, Any], + model_id: str, + decoded_token: Dict[str, Any], + ) -> CompressionResult: + """ + Perform the actual compression operation. + + Args: + conversation_id: Conversation ID + conversation: Conversation document + model_id: Model ID for conversation + decoded_token: User token + + Returns: + CompressionResult + """ + try: + # Determine which model to use for compression + compression_model = ( + settings.COMPRESSION_MODEL_OVERRIDE + if settings.COMPRESSION_MODEL_OVERRIDE + else model_id + ) + + # Get provider and API key for compression model + provider = get_provider_from_model_id(compression_model) + api_key = get_api_key_for_provider(provider) + + # Create compression LLM + compression_llm = LLMCreator.create_llm( + provider, + api_key=api_key, + user_api_key=None, + decoded_token=decoded_token, + model_id=compression_model, + ) + + # Create compression service with DB update capability + compression_service = CompressionService( + llm=compression_llm, + model_id=compression_model, + conversation_service=self.conversation_service, + ) + + # Compress all queries up to the latest + queries_count = len(conversation.get("queries", [])) + compress_up_to = queries_count - 1 + + if compress_up_to < 0: + logger.warning("No queries to compress") + return CompressionResult.success_no_compression([]) + + logger.info( + f"Initiating compression for conversation {conversation_id}: " + f"compressing all {queries_count} queries (0-{compress_up_to})" + ) + + # Perform compression and save to DB + metadata = compression_service.compress_and_save( + conversation_id, conversation, compress_up_to + ) + + logger.info( + f"Compression successful - ratio: {metadata.compression_ratio:.1f}x, " + f"saved {metadata.original_token_count - metadata.compressed_token_count} tokens" + ) + + # Reload conversation with updated metadata + conversation = self.conversation_service.get_conversation( + conversation_id, user_id=decoded_token.get("sub") + ) + + # Get compressed context + compressed_summary, recent_queries = ( + compression_service.get_compressed_context(conversation) + ) + + return CompressionResult.success_with_compression( + compressed_summary, recent_queries, metadata + ) + + except Exception as e: + logger.error(f"Error performing compression: {str(e)}", exc_info=True) + return CompressionResult.failure(str(e)) + + def compress_mid_execution( + self, + conversation_id: str, + user_id: str, + model_id: str, + decoded_token: Dict[str, Any], + current_conversation: Optional[Dict[str, Any]] = None, + ) -> CompressionResult: + """ + Perform compression during tool execution. + + Args: + conversation_id: Conversation ID + user_id: User ID + model_id: Model ID + decoded_token: User token + current_conversation: Pre-loaded conversation (optional) + + Returns: + CompressionResult + """ + try: + # Load conversation if not provided + if current_conversation: + conversation = current_conversation + else: + conversation = self.conversation_service.get_conversation( + conversation_id, user_id + ) + + if not conversation: + logger.warning( + f"Could not load conversation {conversation_id} for mid-execution compression" + ) + return CompressionResult.failure("Conversation not found") + + # Perform compression + return self._perform_compression( + conversation_id, conversation, model_id, decoded_token + ) + + except Exception as e: + logger.error( + f"Error in mid-execution compression: {str(e)}", exc_info=True + ) + return CompressionResult.failure(str(e)) diff --git a/application/api/answer/services/compression/prompt_builder.py b/application/api/answer/services/compression/prompt_builder.py new file mode 100644 index 00000000..d5ce3183 --- /dev/null +++ b/application/api/answer/services/compression/prompt_builder.py @@ -0,0 +1,149 @@ +"""Compression prompt building logic.""" + +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +class CompressionPromptBuilder: + """Builds prompts for LLM compression calls.""" + + def __init__(self, version: str = "v1.0"): + """ + Initialize prompt builder. + + Args: + version: Prompt template version to use + """ + self.version = version + self.system_prompt = self._load_prompt(version) + + def _load_prompt(self, version: str) -> str: + """ + Load prompt template from file. + + Args: + version: Version string (e.g., 'v1.0') + + Returns: + Prompt template content + + Raises: + FileNotFoundError: If prompt template file doesn't exist + """ + current_dir = Path(__file__).resolve().parents[4] + prompt_path = current_dir / "prompts" / "compression" / f"{version}.txt" + + try: + with open(prompt_path, "r") as f: + return f.read() + except FileNotFoundError: + logger.error(f"Compression prompt template not found: {prompt_path}") + raise FileNotFoundError( + f"Compression prompt template '{version}' not found at {prompt_path}. " + f"Please ensure the template file exists." + ) + + def build_prompt( + self, + queries: List[Dict[str, Any]], + existing_compressions: Optional[List[Dict[str, Any]]] = None, + ) -> List[Dict[str, str]]: + """ + Build messages for compression LLM call. + + Args: + queries: List of query objects to compress + existing_compressions: List of previous compression points + + Returns: + List of message dicts for LLM + """ + # Build conversation text + conversation_text = self._format_conversation(queries) + + # Add existing compression context if present + existing_compression_context = "" + if existing_compressions and len(existing_compressions) > 0: + existing_compression_context = ( + "\n\nIMPORTANT: This conversation has been compressed before. " + "Previous compression summaries:\n\n" + ) + for i, comp in enumerate(existing_compressions): + existing_compression_context += ( + f"--- Compression {i + 1} (up to message {comp.get('query_index', 'unknown')}) ---\n" + f"{comp.get('compressed_summary', '')}\n\n" + ) + existing_compression_context += ( + "Your task is to create a NEW summary that incorporates the context from " + "previous compressions AND the new messages below. The final summary should " + "be comprehensive and include all important information from both previous " + "compressions and new messages.\n\n" + ) + + user_prompt = ( + f"{existing_compression_context}" + f"Here is the conversation to summarize:\n\n" + f"{conversation_text}" + ) + + messages = [ + {"role": "system", "content": self.system_prompt}, + {"role": "user", "content": user_prompt}, + ] + + return messages + + def _format_conversation(self, queries: List[Dict[str, Any]]) -> str: + """ + Format conversation queries into readable text for compression. + + Args: + queries: List of query objects + + Returns: + Formatted conversation text + """ + conversation_lines = [] + + for i, query in enumerate(queries): + conversation_lines.append(f"--- Message {i + 1} ---") + conversation_lines.append(f"User: {query.get('prompt', '')}") + + # Add tool calls if present + tool_calls = query.get("tool_calls", []) + if tool_calls: + conversation_lines.append("\nTool Calls:") + for tc in tool_calls: + tool_name = tc.get("tool_name", "unknown") + action_name = tc.get("action_name", "unknown") + arguments = tc.get("arguments", {}) + result = tc.get("result", "") + if result is None: + result = "" + status = tc.get("status", "unknown") + + # Include full tool result for complete compression context + conversation_lines.append( + f" - {tool_name}.{action_name}({arguments}) " + f"[{status}] → {result}" + ) + + # Add agent thought if present + thought = query.get("thought", "") + if thought: + conversation_lines.append(f"\nAgent Thought: {thought}") + + # Add assistant response + conversation_lines.append(f"\nAssistant: {query.get('response', '')}") + + # Add sources if present + sources = query.get("sources", []) + if sources: + conversation_lines.append(f"\nSources Used: {len(sources)} documents") + + conversation_lines.append("") # Empty line between messages + + return "\n".join(conversation_lines) diff --git a/application/api/answer/services/compression/service.py b/application/api/answer/services/compression/service.py new file mode 100644 index 00000000..ccf6f126 --- /dev/null +++ b/application/api/answer/services/compression/service.py @@ -0,0 +1,306 @@ +"""Core compression service with simplified responsibilities.""" + +import logging +import re +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from application.api.answer.services.compression.prompt_builder import ( + CompressionPromptBuilder, +) +from application.api.answer.services.compression.token_counter import TokenCounter +from application.api.answer.services.compression.types import ( + CompressionMetadata, +) +from application.core.settings import settings + +logger = logging.getLogger(__name__) + + +class CompressionService: + """ + Service for compressing conversation history. + + Handles DB updates. + """ + + def __init__( + self, + llm, + model_id: str, + conversation_service=None, + prompt_builder: Optional[CompressionPromptBuilder] = None, + ): + """ + Initialize compression service. + + Args: + llm: LLM instance to use for compression + model_id: Model ID for compression + conversation_service: Service for DB operations (optional, for DB updates) + prompt_builder: Custom prompt builder (optional) + """ + self.llm = llm + self.model_id = model_id + self.conversation_service = conversation_service + self.prompt_builder = prompt_builder or CompressionPromptBuilder( + version=settings.COMPRESSION_PROMPT_VERSION + ) + + def compress_conversation( + self, + conversation: Dict[str, Any], + compress_up_to_index: int, + ) -> CompressionMetadata: + """ + Compress conversation history up to specified index. + + Args: + conversation: Full conversation document + compress_up_to_index: Last query index to include in compression + + Returns: + CompressionMetadata with compression details + + Raises: + ValueError: If compress_up_to_index is invalid + """ + try: + queries = conversation.get("queries", []) + + if compress_up_to_index < 0 or compress_up_to_index >= len(queries): + raise ValueError( + f"Invalid compress_up_to_index: {compress_up_to_index} " + f"(conversation has {len(queries)} queries)" + ) + + # Get queries to compress + queries_to_compress = queries[: compress_up_to_index + 1] + + # Check if there are existing compressions + existing_compressions = conversation.get("compression_metadata", {}).get( + "compression_points", [] + ) + + if existing_compressions: + logger.info( + f"Found {len(existing_compressions)} previous compression(s) - " + f"will incorporate into new summary" + ) + + # Calculate original token count + original_tokens = TokenCounter.count_query_tokens(queries_to_compress) + + # Log tool call stats + self._log_tool_call_stats(queries_to_compress) + + # Build compression prompt + messages = self.prompt_builder.build_prompt( + queries_to_compress, existing_compressions + ) + + # Call LLM to generate compression + logger.info( + f"Starting compression: {len(queries_to_compress)} queries " + f"(messages 0-{compress_up_to_index}, {original_tokens} tokens) " + f"using model {self.model_id}" + ) + + response = self.llm.gen( + model=self.model_id, messages=messages, max_tokens=4000 + ) + + # Extract summary from response + compressed_summary = self._extract_summary(response) + + # Calculate compressed token count + compressed_tokens = TokenCounter.count_message_tokens( + [{"content": compressed_summary}] + ) + + # Calculate compression ratio + compression_ratio = ( + original_tokens / compressed_tokens if compressed_tokens > 0 else 0 + ) + + logger.info( + f"Compression complete: {original_tokens} → {compressed_tokens} tokens " + f"({compression_ratio:.1f}x compression)" + ) + + # Build compression metadata + compression_metadata = CompressionMetadata( + timestamp=datetime.now(timezone.utc), + query_index=compress_up_to_index, + compressed_summary=compressed_summary, + original_token_count=original_tokens, + compressed_token_count=compressed_tokens, + compression_ratio=compression_ratio, + model_used=self.model_id, + compression_prompt_version=self.prompt_builder.version, + ) + + return compression_metadata + + except Exception as e: + logger.error(f"Error compressing conversation: {str(e)}", exc_info=True) + raise + + def compress_and_save( + self, + conversation_id: str, + conversation: Dict[str, Any], + compress_up_to_index: int, + ) -> CompressionMetadata: + """ + Compress conversation and save to database. + + Args: + conversation_id: Conversation ID + conversation: Full conversation document + compress_up_to_index: Last query index to include + + Returns: + CompressionMetadata + + Raises: + ValueError: If conversation_service not provided or invalid index + """ + if not self.conversation_service: + raise ValueError( + "conversation_service required for compress_and_save operation" + ) + + # Perform compression + metadata = self.compress_conversation(conversation, compress_up_to_index) + + # Save to database + self.conversation_service.update_compression_metadata( + conversation_id, metadata.to_dict() + ) + + logger.info(f"Compression metadata saved to database for {conversation_id}") + + return metadata + + def get_compressed_context( + self, conversation: Dict[str, Any] + ) -> tuple[Optional[str], List[Dict[str, Any]]]: + """ + Get compressed summary + recent uncompressed messages. + + Args: + conversation: Full conversation document + + Returns: + (compressed_summary, recent_messages) + """ + try: + compression_metadata = conversation.get("compression_metadata", {}) + + if not compression_metadata.get("is_compressed"): + logger.debug("No compression metadata found - using full history") + queries = conversation.get("queries", []) + if queries is None: + logger.error("Conversation queries is None - returning empty list") + return None, [] + return None, queries + + compression_points = compression_metadata.get("compression_points", []) + + if not compression_points: + logger.debug("No compression points found - using full history") + queries = conversation.get("queries", []) + if queries is None: + logger.error("Conversation queries is None - returning empty list") + return None, [] + return None, queries + + # Get the most recent compression point + latest_compression = compression_points[-1] + compressed_summary = latest_compression.get("compressed_summary") + last_compressed_index = latest_compression.get("query_index") + compressed_tokens = latest_compression.get("compressed_token_count", 0) + original_tokens = latest_compression.get("original_token_count", 0) + + # Get only messages after compression point + queries = conversation.get("queries", []) + total_queries = len(queries) + recent_queries = queries[last_compressed_index + 1 :] + + logger.info( + f"Using compressed context: summary ({compressed_tokens} tokens, " + f"compressed from {original_tokens}) + {len(recent_queries)} recent messages " + f"(messages {last_compressed_index + 1}-{total_queries - 1})" + ) + + return compressed_summary, recent_queries + + except Exception as e: + logger.error( + f"Error getting compressed context: {str(e)}", exc_info=True + ) + queries = conversation.get("queries", []) + if queries is None: + return None, [] + return None, queries + + def _extract_summary(self, llm_response: str) -> str: + """ + Extract clean summary from LLM response. + + Args: + llm_response: Raw LLM response + + Returns: + Cleaned summary text + """ + try: + # Try to extract content within tags + summary_match = re.search( + r"(.*?)", llm_response, re.DOTALL + ) + + if summary_match: + summary = summary_match.group(1).strip() + else: + # If no summary tags, remove analysis tags and use the rest + summary = re.sub( + r".*?", "", llm_response, flags=re.DOTALL + ).strip() + + return summary + + except Exception as e: + logger.warning(f"Error extracting summary: {str(e)}, using full response") + return llm_response + + def _log_tool_call_stats(self, queries: List[Dict[str, Any]]) -> None: + """Log statistics about tool calls in queries.""" + total_tool_calls = 0 + total_tool_result_chars = 0 + tool_call_breakdown = {} + + for q in queries: + for tc in q.get("tool_calls", []): + total_tool_calls += 1 + tool_name = tc.get("tool_name", "unknown") + action_name = tc.get("action_name", "unknown") + key = f"{tool_name}.{action_name}" + tool_call_breakdown[key] = tool_call_breakdown.get(key, 0) + 1 + + # Track total tool result size + result = tc.get("result", "") + if result: + total_tool_result_chars += len(str(result)) + + if total_tool_calls > 0: + tool_breakdown_str = ", ".join( + f"{tool}({count})" + for tool, count in sorted(tool_call_breakdown.items()) + ) + tool_result_kb = total_tool_result_chars / 1024 + logger.info( + f"Tool call breakdown: {tool_breakdown_str} " + f"(total result size: {tool_result_kb:.1f} KB, {total_tool_result_chars:,} chars)" + ) diff --git a/application/api/answer/services/compression/threshold_checker.py b/application/api/answer/services/compression/threshold_checker.py new file mode 100644 index 00000000..15397018 --- /dev/null +++ b/application/api/answer/services/compression/threshold_checker.py @@ -0,0 +1,103 @@ +"""Compression threshold checking logic.""" + +import logging +from typing import Any, Dict + +from application.core.model_utils import get_token_limit +from application.core.settings import settings +from application.api.answer.services.compression.token_counter import TokenCounter + +logger = logging.getLogger(__name__) + + +class CompressionThresholdChecker: + """Determines if compression is needed based on token thresholds.""" + + def __init__(self, threshold_percentage: float = None): + """ + Initialize threshold checker. + + Args: + threshold_percentage: Percentage of context to use as threshold + (defaults to settings.COMPRESSION_THRESHOLD_PERCENTAGE) + """ + self.threshold_percentage = ( + threshold_percentage or settings.COMPRESSION_THRESHOLD_PERCENTAGE + ) + + def should_compress( + self, + conversation: Dict[str, Any], + model_id: str, + current_query_tokens: int = 500, + ) -> bool: + """ + Determine if compression is needed. + + Args: + conversation: Full conversation document + model_id: Target model for this request + current_query_tokens: Estimated tokens for current query + + Returns: + True if tokens >= threshold% of context window + """ + try: + # Calculate total tokens in conversation + total_tokens = TokenCounter.count_conversation_tokens(conversation) + total_tokens += current_query_tokens + + # Get context window limit for model + context_limit = get_token_limit(model_id) + + # Calculate threshold + threshold = int(context_limit * self.threshold_percentage) + + compression_needed = total_tokens >= threshold + percentage_used = (total_tokens / context_limit) * 100 + + if compression_needed: + logger.warning( + f"COMPRESSION TRIGGERED: {total_tokens} tokens / {context_limit} limit " + f"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%)" + ) + else: + logger.info( + f"Compression check: {total_tokens}/{context_limit} tokens " + f"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%) - No compression needed" + ) + + return compression_needed + + except Exception as e: + logger.error(f"Error checking compression need: {str(e)}", exc_info=True) + return False + + def check_message_tokens(self, messages: list, model_id: str) -> bool: + """ + Check if message list exceeds threshold. + + Args: + messages: List of message dicts + model_id: Target model + + Returns: + True if at or above threshold + """ + try: + current_tokens = TokenCounter.count_message_tokens(messages) + context_limit = get_token_limit(model_id) + threshold = int(context_limit * self.threshold_percentage) + + if current_tokens >= threshold: + logger.warning( + f"Message context limit approaching: {current_tokens}/{context_limit} tokens " + f"({(current_tokens/context_limit)*100:.1f}%)" + ) + return True + + return False + + except Exception as e: + logger.error(f"Error checking message tokens: {str(e)}", exc_info=True) + return False diff --git a/application/api/answer/services/compression/token_counter.py b/application/api/answer/services/compression/token_counter.py new file mode 100644 index 00000000..ac676cf0 --- /dev/null +++ b/application/api/answer/services/compression/token_counter.py @@ -0,0 +1,103 @@ +"""Token counting utilities for compression.""" + +import logging +from typing import Any, Dict, List + +from application.utils import num_tokens_from_string +from application.core.settings import settings + +logger = logging.getLogger(__name__) + + +class TokenCounter: + """Centralized token counting for conversations and messages.""" + + @staticmethod + def count_message_tokens(messages: List[Dict]) -> int: + """ + Calculate total tokens in a list of messages. + + Args: + messages: List of message dicts with 'content' field + + Returns: + Total token count + """ + total_tokens = 0 + for message in messages: + content = message.get("content", "") + if isinstance(content, str): + total_tokens += num_tokens_from_string(content) + elif isinstance(content, list): + # Handle structured content (tool calls, etc.) + for item in content: + if isinstance(item, dict): + total_tokens += num_tokens_from_string(str(item)) + return total_tokens + + @staticmethod + def count_query_tokens( + queries: List[Dict[str, Any]], include_tool_calls: bool = True + ) -> int: + """ + Count tokens across multiple query objects. + + Args: + queries: List of query objects from conversation + include_tool_calls: Whether to count tool call tokens + + Returns: + Total token count + """ + total_tokens = 0 + + for query in queries: + # Count prompt and response tokens + if "prompt" in query: + total_tokens += num_tokens_from_string(query["prompt"]) + if "response" in query: + total_tokens += num_tokens_from_string(query["response"]) + if "thought" in query: + total_tokens += num_tokens_from_string(query.get("thought", "")) + + # Count tool call tokens + if include_tool_calls and "tool_calls" in query: + for tool_call in query["tool_calls"]: + tool_call_string = ( + f"Tool: {tool_call.get('tool_name')} | " + f"Action: {tool_call.get('action_name')} | " + f"Args: {tool_call.get('arguments')} | " + f"Response: {tool_call.get('result')}" + ) + total_tokens += num_tokens_from_string(tool_call_string) + + return total_tokens + + @staticmethod + def count_conversation_tokens( + conversation: Dict[str, Any], include_system_prompt: bool = False + ) -> int: + """ + Calculate total tokens in a conversation. + + Args: + conversation: Conversation document + include_system_prompt: Whether to include system prompt in count + + Returns: + Total token count + """ + try: + queries = conversation.get("queries", []) + total_tokens = TokenCounter.count_query_tokens(queries) + + # Add system prompt tokens if requested + if include_system_prompt: + # Rough estimate for system prompt + total_tokens += settings.RESERVED_TOKENS.get("system_prompt", 500) + + return total_tokens + + except Exception as e: + logger.error(f"Error calculating conversation tokens: {str(e)}") + return 0 diff --git a/application/api/answer/services/compression/types.py b/application/api/answer/services/compression/types.py new file mode 100644 index 00000000..b71ab9ee --- /dev/null +++ b/application/api/answer/services/compression/types.py @@ -0,0 +1,83 @@ +"""Type definitions for compression module.""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Dict, List, Optional + + +@dataclass +class CompressionMetadata: + """Metadata about a compression operation.""" + + timestamp: datetime + query_index: int + compressed_summary: str + original_token_count: int + compressed_token_count: int + compression_ratio: float + model_used: str + compression_prompt_version: str + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for DB storage.""" + return { + "timestamp": self.timestamp, + "query_index": self.query_index, + "compressed_summary": self.compressed_summary, + "original_token_count": self.original_token_count, + "compressed_token_count": self.compressed_token_count, + "compression_ratio": self.compression_ratio, + "model_used": self.model_used, + "compression_prompt_version": self.compression_prompt_version, + } + + +@dataclass +class CompressionResult: + """Result of a compression operation.""" + + success: bool + compressed_summary: Optional[str] = None + recent_queries: List[Dict[str, Any]] = field(default_factory=list) + metadata: Optional[CompressionMetadata] = None + error: Optional[str] = None + compression_performed: bool = False + + @classmethod + def success_with_compression( + cls, summary: str, queries: List[Dict], metadata: CompressionMetadata + ) -> "CompressionResult": + """Create a successful result with compression.""" + return cls( + success=True, + compressed_summary=summary, + recent_queries=queries, + metadata=metadata, + compression_performed=True, + ) + + @classmethod + def success_no_compression(cls, queries: List[Dict]) -> "CompressionResult": + """Create a successful result without compression needed.""" + return cls( + success=True, + recent_queries=queries, + compression_performed=False, + ) + + @classmethod + def failure(cls, error: str) -> "CompressionResult": + """Create a failure result.""" + return cls(success=False, error=error, compression_performed=False) + + def as_history(self) -> List[Dict[str, str]]: + """ + Convert recent queries to history format. + + Returns: + List of prompt/response dicts + """ + return [ + {"prompt": q["prompt"], "response": q["response"]} + for q in self.recent_queries + ] diff --git a/application/api/answer/services/conversation_service.py b/application/api/answer/services/conversation_service.py index 0e98983e..bf55801c 100644 --- a/application/api/answer/services/conversation_service.py +++ b/application/api/answer/services/conversation_service.py @@ -180,3 +180,103 @@ class ConversationService: conversation_data["api_key"] = agent["key"] result = self.conversations_collection.insert_one(conversation_data) return str(result.inserted_id) + + def update_compression_metadata( + self, conversation_id: str, compression_metadata: Dict[str, Any] + ) -> None: + """ + Update conversation with compression metadata. + + Uses $push with $slice to keep only the most recent compression points, + preventing unbounded array growth. Since each compression incorporates + previous compressions, older points become redundant. + + Args: + conversation_id: Conversation ID + compression_metadata: Compression point data + """ + try: + self.conversations_collection.update_one( + {"_id": ObjectId(conversation_id)}, + { + "$set": { + "compression_metadata.is_compressed": True, + "compression_metadata.last_compression_at": compression_metadata.get( + "timestamp" + ), + }, + "$push": { + "compression_metadata.compression_points": { + "$each": [compression_metadata], + "$slice": -settings.COMPRESSION_MAX_HISTORY_POINTS, + } + }, + }, + ) + logger.info( + f"Updated compression metadata for conversation {conversation_id}" + ) + except Exception as e: + logger.error( + f"Error updating compression metadata: {str(e)}", exc_info=True + ) + raise + + def append_compression_message( + self, conversation_id: str, compression_metadata: Dict[str, Any] + ) -> None: + """ + Append a synthetic compression summary entry into the conversation history. + This makes the summary visible in the DB alongside normal queries. + """ + try: + summary = compression_metadata.get("compressed_summary", "") + if not summary: + return + timestamp = compression_metadata.get("timestamp", datetime.now(timezone.utc)) + + self.conversations_collection.update_one( + {"_id": ObjectId(conversation_id)}, + { + "$push": { + "queries": { + "prompt": "[Context Compression Summary]", + "response": summary, + "thought": "", + "sources": [], + "tool_calls": [], + "timestamp": timestamp, + "attachments": [], + "model_id": compression_metadata.get("model_used"), + } + } + }, + ) + logger.info(f"Appended compression summary to conversation {conversation_id}") + except Exception as e: + logger.error( + f"Error appending compression summary: {str(e)}", exc_info=True + ) + + def get_compression_metadata( + self, conversation_id: str + ) -> Optional[Dict[str, Any]]: + """ + Get compression metadata for a conversation. + + Args: + conversation_id: Conversation ID + + Returns: + Compression metadata dict or None + """ + try: + conversation = self.conversations_collection.find_one( + {"_id": ObjectId(conversation_id)}, {"compression_metadata": 1} + ) + return conversation.get("compression_metadata") if conversation else None + except Exception as e: + logger.error( + f"Error getting compression metadata: {str(e)}", exc_info=True + ) + return None diff --git a/application/api/answer/services/stream_processor.py b/application/api/answer/services/stream_processor.py index 586e7696..5d97fbe8 100644 --- a/application/api/answer/services/stream_processor.py +++ b/application/api/answer/services/stream_processor.py @@ -10,6 +10,8 @@ from bson.dbref import DBRef from bson.objectid import ObjectId from application.agents.agent_creator import AgentCreator +from application.api.answer.services.compression import CompressionOrchestrator +from application.api.answer.services.compression.token_counter import TokenCounter from application.api.answer.services.conversation_service import ConversationService from application.api.answer.services.prompt_renderer import PromptRenderer from application.core.model_utils import ( @@ -90,9 +92,14 @@ class StreamProcessor: self.shared_token = None self.model_id: Optional[str] = None self.conversation_service = ConversationService() + self.compression_orchestrator = CompressionOrchestrator( + self.conversation_service + ) self.prompt_renderer = PromptRenderer() self._prompt_content: Optional[str] = None self._required_tool_actions: Optional[Dict[str, Set[Optional[str]]]] = None + self.compressed_summary: Optional[str] = None + self.compressed_summary_tokens: int = 0 def initialize(self): """Initialize all required components for processing""" @@ -112,15 +119,72 @@ class StreamProcessor: ) if not conversation: raise ValueError("Conversation not found or unauthorized") - self.history = [ - {"prompt": query["prompt"], "response": query["response"]} - for query in conversation.get("queries", []) - ] + + # Check if compression is enabled and needed + if settings.ENABLE_CONVERSATION_COMPRESSION: + self._handle_compression(conversation) + else: + # Original behavior - load all history + self.history = [ + {"prompt": query["prompt"], "response": query["response"]} + for query in conversation.get("queries", []) + ] else: self.history = limit_chat_history( json.loads(self.data.get("history", "[]")), model_id=self.model_id ) + def _handle_compression(self, conversation: Dict[str, Any]): + """ + Handle conversation compression logic using orchestrator. + + Args: + conversation: Full conversation document + """ + try: + # Use orchestrator to handle all compression logic + result = self.compression_orchestrator.compress_if_needed( + conversation_id=self.conversation_id, + user_id=self.initial_user_id, + model_id=self.model_id, + decoded_token=self.decoded_token, + ) + + if not result.success: + logger.error( + f"Compression failed: {result.error}, using full history" + ) + self.history = [ + {"prompt": query["prompt"], "response": query["response"]} + for query in conversation.get("queries", []) + ] + return + + # Set compressed summary if compression was performed + if result.compression_performed and result.compressed_summary: + self.compressed_summary = result.compressed_summary + self.compressed_summary_tokens = TokenCounter.count_message_tokens( + [{"content": result.compressed_summary}] + ) + logger.info( + f"Using compressed summary ({self.compressed_summary_tokens} tokens) " + f"+ {len(result.recent_queries)} recent messages" + ) + + # Build history from recent queries + self.history = result.as_history() + + except Exception as e: + logger.error( + f"Error handling compression, falling back to standard history: {str(e)}", + exc_info=True, + ) + # Fallback to original behavior + self.history = [ + {"prompt": query["prompt"], "response": query["response"]} + for query in conversation.get("queries", []) + ] + def _process_attachments(self): """Process any attachments in the request""" attachment_ids = self.data.get("attachments", []) @@ -658,7 +722,7 @@ class StreamProcessor: ) system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER) - return AgentCreator.create_agent( + agent = AgentCreator.create_agent( self.agent_config["agent_type"], endpoint="stream", llm_name=provider or settings.LLM_PROVIDER, @@ -671,4 +735,10 @@ class StreamProcessor: decoded_token=self.decoded_token, attachments=self.attachments, json_schema=self.agent_config.get("json_schema"), + compressed_summary=self.compressed_summary, ) + + agent.conversation_id = self.conversation_id + agent.initial_user_id = self.initial_user_id + + return agent diff --git a/application/core/model_configs.py b/application/core/model_configs.py index b802ee27..5f75bc83 100644 --- a/application/core/model_configs.py +++ b/application/core/model_configs.py @@ -29,63 +29,29 @@ GOOGLE_ATTACHMENTS = [ OPENAI_MODELS = [ AvailableModel( - id="gpt-4o", + id="gpt-5.1", provider=ModelProvider.OPENAI, - display_name="GPT-4 Omni", - description="Latest and most capable model", + display_name="GPT-5.1", + description="Flagship model with enhanced reasoning, coding, and agentic capabilities", capabilities=ModelCapabilities( supports_tools=True, supports_structured_output=True, supported_attachment_types=OPENAI_ATTACHMENTS, - context_window=128000, + context_window=400000, ), ), AvailableModel( - id="gpt-4o-mini", + id="gpt-5-mini", provider=ModelProvider.OPENAI, - display_name="GPT-4 Omni Mini", - description="Fast and efficient", + display_name="GPT-5 Mini", + description="Faster, cost-effective variant of GPT-5.1", capabilities=ModelCapabilities( supports_tools=True, supports_structured_output=True, supported_attachment_types=OPENAI_ATTACHMENTS, - context_window=128000, + context_window=400000, ), - ), - AvailableModel( - id="gpt-4-turbo", - provider=ModelProvider.OPENAI, - display_name="GPT-4 Turbo", - description="Fast GPT-4 with 128k context", - capabilities=ModelCapabilities( - supports_tools=True, - supports_structured_output=True, - supported_attachment_types=OPENAI_ATTACHMENTS, - context_window=128000, - ), - ), - AvailableModel( - id="gpt-4", - provider=ModelProvider.OPENAI, - display_name="GPT-4", - description="Most capable model", - capabilities=ModelCapabilities( - supports_tools=True, - supports_structured_output=True, - supported_attachment_types=OPENAI_ATTACHMENTS, - context_window=8192, - ), - ), - AvailableModel( - id="gpt-3.5-turbo", - provider=ModelProvider.OPENAI, - display_name="GPT-3.5 Turbo", - description="Fast and cost-effective", - capabilities=ModelCapabilities( - supports_tools=True, - context_window=4096, - ), - ), + ) ] @@ -159,15 +125,15 @@ GOOGLE_MODELS = [ ), ), AvailableModel( - id="gemini-2.5-pro", + id="gemini-3-pro-preview", provider=ModelProvider.GOOGLE, - display_name="Gemini 2.5 Pro", + display_name="Gemini 3 Pro", description="Most capable Gemini model", capabilities=ModelCapabilities( supports_tools=True, supports_structured_output=True, supported_attachment_types=GOOGLE_ATTACHMENTS, - context_window=2000000, + context_window=20000, # Set low for testing compression ), ), ] diff --git a/application/core/settings.py b/application/core/settings.py index ee7ffa05..cecbb333 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -144,6 +144,13 @@ class Settings(BaseSettings): # Tool pre-fetch settings ENABLE_TOOL_PREFETCH: bool = True + # Conversation Compression Settings + ENABLE_CONVERSATION_COMPRESSION: bool = True + COMPRESSION_THRESHOLD_PERCENTAGE: float = 0.8 # Trigger at 80% of context + COMPRESSION_MODEL_OVERRIDE: Optional[str] = None # Use different model for compression + COMPRESSION_PROMPT_VERSION: str = "v1.0" # Track prompt iterations + COMPRESSION_MAX_HISTORY_POINTS: int = 3 # Keep only last N compression points to prevent DB bloat + path = Path(__file__).parent.parent.absolute() settings = Settings(_env_file=path.joinpath(".env"), _env_file_encoding="utf-8") diff --git a/application/llm/google_ai.py b/application/llm/google_ai.py index 9c58a3e1..00c609ec 100644 --- a/application/llm/google_ai.py +++ b/application/llm/google_ai.py @@ -1,4 +1,3 @@ -import json import logging from google import genai @@ -11,11 +10,13 @@ from application.storage.storage_creator import StorageCreator class GoogleLLM(BaseLLM): - def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): + def __init__( + self, api_key=None, user_api_key=None, decoded_token=None, *args, **kwargs + ): super().__init__(*args, **kwargs) self.api_key = api_key or settings.GOOGLE_API_KEY or settings.API_KEY self.user_api_key = user_api_key - + self.client = genai.Client(api_key=self.api_key) self.storage = StorageCreator.get_storage() @@ -33,6 +34,12 @@ class GoogleLLM(BaseLLM): "image/jpg", "image/webp", "image/gif", + "application/pdf", + "image/png", + "image/jpeg", + "image/jpg", + "image/webp", + "image/gif", ] def prepare_messages_with_attachments(self, messages, attachments=None): @@ -135,12 +142,38 @@ class GoogleLLM(BaseLLM): raise def _clean_messages_google(self, messages): - """Convert OpenAI format messages to Google AI format.""" + """ + Convert OpenAI format messages to Google AI format and collect system prompts. + + Returns: + tuple[list[types.Content], Optional[str]]: cleaned messages and optional + combined system instruction. + """ cleaned_messages = [] + system_instructions = [] + + def _extract_system_text(content): + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for item in content: + if isinstance(item, dict) and "text" in item and item["text"] is not None: + parts.append(item["text"]) + return "\n".join(parts) + return "" + for message in messages: role = message.get("role") content = message.get("content") + # Gemini only accepts user/model in the contents list. + if role == "system": + sys_text = _extract_system_text(content) + if sys_text: + system_instructions.append(sys_text) + continue + if role == "assistant": role = "model" elif role == "tool": @@ -159,12 +192,27 @@ class GoogleLLM(BaseLLM): cleaned_args = self._remove_null_values( item["function_call"]["args"] ) - parts.append( - types.Part.from_function_call( - name=item["function_call"]["name"], - args=cleaned_args, + # Create function call part with thought_signature if present + # For Gemini 3 models, we need to include thought_signature + if "thought_signature" in item: + # Use Part constructor with functionCall and thoughtSignature + parts.append( + types.Part( + functionCall=types.FunctionCall( + name=item["function_call"]["name"], + args=cleaned_args, + ), + thoughtSignature=item["thought_signature"], + ) + ) + else: + # Use helper method when no thought_signature + parts.append( + types.Part.from_function_call( + name=item["function_call"]["name"], + args=cleaned_args, + ) ) - ) elif "function_response" in item: parts.append( types.Part.from_function_response( @@ -188,7 +236,8 @@ class GoogleLLM(BaseLLM): raise ValueError(f"Unexpected content type: {type(content)}") if parts: cleaned_messages.append(types.Content(role=role, parts=parts)) - return cleaned_messages + system_instruction = "\n\n".join(system_instructions) if system_instructions else None + return cleaned_messages, system_instruction def _clean_schema(self, schema_obj): """ @@ -274,6 +323,61 @@ class GoogleLLM(BaseLLM): genai_tools.append(genai_tool) return genai_tools + def _extract_preview_from_message(self, message): + """Get a short, human-readable preview from the last message.""" + try: + if hasattr(message, "parts"): + for part in reversed(message.parts): + if getattr(part, "text", None): + return part.text + function_call = getattr(part, "function_call", None) + if function_call: + name = getattr(function_call, "name", "") or "function_call" + return f"function_call:{name}" + function_response = getattr(part, "function_response", None) + if function_response: + name = getattr(function_response, "name", "") or "function_response" + return f"function_response:{name}" + if isinstance(message, dict): + content = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + for item in reversed(content): + if isinstance(item, str): + return item + if isinstance(item, dict): + if item.get("text"): + return item["text"] + if item.get("function_call"): + fn = item["function_call"] + if isinstance(fn, dict): + name = fn.get("name") or "function_call" + return f"function_call:{name}" + return "function_call" + if item.get("function_response"): + resp = item["function_response"] + if isinstance(resp, dict): + name = resp.get("name") or "function_response" + return f"function_response:{name}" + return "function_response" + if "text" in message and isinstance(message["text"], str): + return message["text"] + except Exception: + pass + return str(message) + + def _summarize_messages_for_log(self, messages, preview_chars=20): + """Return a compact summary for logging to avoid huge payloads.""" + message_count = len(messages) if messages else 0 + last_preview = "" + if messages: + last_preview = self._extract_preview_from_message(messages[-1]) or "" + last_preview = str(last_preview).replace("\n", " ") + if len(last_preview) > preview_chars: + last_preview = f"{last_preview[:preview_chars]}..." + return f"count={message_count}, last='{last_preview}'" + def _raw_gen( self, baseself, @@ -287,12 +391,12 @@ class GoogleLLM(BaseLLM): ): """Generate content using Google AI API without streaming.""" client = genai.Client(api_key=self.api_key) + system_instruction = None if formatting == "openai": - messages = self._clean_messages_google(messages) + messages, system_instruction = self._clean_messages_google(messages) config = types.GenerateContentConfig() - if messages[0].role == "system": - config.system_instruction = messages[0].parts[0].text - messages = messages[1:] + if system_instruction: + config.system_instruction = system_instruction if tools: cleaned_tools = self._clean_tools_format(tools) config.tools = cleaned_tools @@ -325,12 +429,12 @@ class GoogleLLM(BaseLLM): ): """Generate content using Google AI API with streaming.""" client = genai.Client(api_key=self.api_key) + system_instruction = None if formatting == "openai": - messages = self._clean_messages_google(messages) + messages, system_instruction = self._clean_messages_google(messages) config = types.GenerateContentConfig() - if messages[0].role == "system": - config.system_instruction = messages[0].parts[0].text - messages = messages[1:] + if system_instruction: + config.system_instruction = system_instruction if tools: cleaned_tools = self._clean_tools_format(tools) config.tools = cleaned_tools @@ -349,8 +453,12 @@ class GoogleLLM(BaseLLM): break if has_attachments: break + messages_summary = self._summarize_messages_for_log(messages) logging.info( - f"GoogleLLM: Starting stream generation. Model: {model}, Messages: {json.dumps(messages, default=str)}, Has attachments: {has_attachments}" + "GoogleLLM: Starting stream generation. Model: %s, Messages: %s, Has attachments: %s", + model, + messages_summary, + has_attachments, ) response = client.models.generate_content_stream( diff --git a/application/llm/handlers/base.py b/application/llm/handlers/base.py index 920caf65..b11654c5 100644 --- a/application/llm/handlers/base.py +++ b/application/llm/handlers/base.py @@ -1,4 +1,5 @@ import logging +import uuid from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, Dict, Generator, List, Optional, Union @@ -16,6 +17,7 @@ class ToolCall: name: str arguments: Union[str, Dict] index: Optional[int] = None + thought_signature: Optional[str] = None @classmethod def from_dict(cls, data: Dict) -> "ToolCall": @@ -178,6 +180,406 @@ class LLMHandler(ABC): system_msg["content"] += f"\n\n{combined_text}" return prepared_messages + def _prune_messages_minimal(self, messages: List[Dict]) -> Optional[List[Dict]]: + """ + Build a minimal context: system prompt + latest user message only. + Drops all tool/function messages to shrink context aggressively. + """ + system_message = next((m for m in messages if m.get("role") == "system"), None) + if not system_message: + logger.warning("Cannot prune messages minimally: missing system message.") + return None + last_non_system = None + for m in reversed(messages): + if m.get("role") == "user": + last_non_system = m + break + if not last_non_system and m.get("role") not in ("system", None): + last_non_system = m + if not last_non_system: + logger.warning("Cannot prune messages minimally: missing user/assistant messages.") + return None + logger.info("Pruning context to system + latest user/assistant message to proceed.") + return [system_message, last_non_system] + + def _extract_text_from_content(self, content: Any) -> str: + """ + Convert message content (str or list of parts) to plain text for compression. + """ + if isinstance(content, str): + return content + if isinstance(content, list): + parts_text = [] + for item in content: + if isinstance(item, dict): + if "text" in item and item["text"] is not None: + parts_text.append(str(item["text"])) + elif "function_call" in item or "function_response" in item: + # Keep serialized function calls/responses so the compressor sees actions + parts_text.append(str(item)) + elif "files" in item: + parts_text.append(str(item)) + return "\n".join(parts_text) + return "" + + def _build_conversation_from_messages(self, messages: List[Dict]) -> Optional[Dict]: + """ + Build a conversation-like dict from current messages so we can compress + even when the conversation isn't persisted yet. Includes tool calls/results. + """ + queries = [] + current_prompt = None + current_tool_calls = {} + + def _commit_query(response_text: str): + nonlocal current_prompt, current_tool_calls + if current_prompt is None and not response_text: + return + tool_calls_list = list(current_tool_calls.values()) + queries.append( + { + "prompt": current_prompt or "", + "response": response_text, + "tool_calls": tool_calls_list, + } + ) + current_prompt = None + current_tool_calls = {} + + for message in messages: + role = message.get("role") + content = message.get("content") + + if role == "user": + current_prompt = self._extract_text_from_content(content) + + elif role in {"assistant", "model"}: + # If this assistant turn contains tool calls, collect them; otherwise commit a response. + if isinstance(content, list): + for item in content: + if "function_call" in item: + fc = item["function_call"] + call_id = fc.get("call_id") or str(uuid.uuid4()) + current_tool_calls[call_id] = { + "tool_name": "unknown_tool", + "action_name": fc.get("name"), + "arguments": fc.get("args"), + "result": None, + "status": "called", + "call_id": call_id, + } + elif "function_response" in item: + fr = item["function_response"] + call_id = fr.get("call_id") or str(uuid.uuid4()) + current_tool_calls[call_id] = { + "tool_name": "unknown_tool", + "action_name": fr.get("name"), + "arguments": None, + "result": fr.get("response", {}).get("result"), + "status": "completed", + "call_id": call_id, + } + # No direct assistant text here; continue to next message + continue + + response_text = self._extract_text_from_content(content) + _commit_query(response_text) + + elif role == "tool": + # Attach tool outputs to the latest pending tool call if possible + tool_text = self._extract_text_from_content(content) + # Attempt to parse function_response style + call_id = None + if isinstance(content, list): + for item in content: + if "function_response" in item and item["function_response"].get("call_id"): + call_id = item["function_response"]["call_id"] + break + if call_id and call_id in current_tool_calls: + current_tool_calls[call_id]["result"] = tool_text + current_tool_calls[call_id]["status"] = "completed" + elif queries: + queries[-1].setdefault("tool_calls", []).append( + { + "tool_name": "unknown_tool", + "action_name": "unknown_action", + "arguments": {}, + "result": tool_text, + "status": "completed", + } + ) + + # If there's an unfinished prompt with tool_calls but no response yet, commit it + if current_prompt is not None or current_tool_calls: + _commit_query(response_text="") + + if not queries: + return None + + return { + "queries": queries, + "compression_metadata": { + "is_compressed": False, + "compression_points": [], + }, + } + + def _rebuild_messages_after_compression( + self, + messages: List[Dict], + compressed_summary: Optional[str], + recent_queries: List[Dict], + include_current_execution: bool = False, + include_tool_calls: bool = False, + ) -> Optional[List[Dict]]: + """ + Rebuild the message list after compression so tool execution can continue. + + Delegates to MessageBuilder for the actual reconstruction. + """ + from application.api.answer.services.compression.message_builder import ( + MessageBuilder, + ) + + return MessageBuilder.rebuild_messages_after_compression( + messages=messages, + compressed_summary=compressed_summary, + recent_queries=recent_queries, + include_current_execution=include_current_execution, + include_tool_calls=include_tool_calls, + ) + + def _perform_mid_execution_compression( + self, agent, messages: List[Dict] + ) -> tuple[bool, Optional[List[Dict]]]: + """ + Perform compression during tool execution and rebuild messages. + + Uses the new orchestrator for simplified compression. + + Args: + agent: The agent instance + messages: Current conversation messages + + Returns: + (success: bool, rebuilt_messages: Optional[List[Dict]]) + """ + try: + from application.api.answer.services.compression import ( + CompressionOrchestrator, + ) + from application.api.answer.services.conversation_service import ( + ConversationService, + ) + + conversation_service = ConversationService() + orchestrator = CompressionOrchestrator(conversation_service) + + # Get conversation from database (may be None for new sessions) + conversation = conversation_service.get_conversation( + agent.conversation_id, agent.initial_user_id + ) + + if conversation: + # Merge current in-flight messages (including tool calls) + conversation_from_msgs = self._build_conversation_from_messages(messages) + if conversation_from_msgs: + conversation = conversation_from_msgs + else: + logger.warning( + "Could not load conversation for compression; attempting in-memory compression" + ) + return self._perform_in_memory_compression(agent, messages) + + # Use orchestrator to perform compression + result = orchestrator.compress_mid_execution( + conversation_id=agent.conversation_id, + user_id=agent.initial_user_id, + model_id=agent.model_id, + decoded_token=getattr(agent, "decoded_token", {}), + current_conversation=conversation, + ) + + if not result.success: + logger.warning(f"Mid-execution compression failed: {result.error}") + # Try minimal pruning as fallback + pruned = self._prune_messages_minimal(messages) + if pruned: + agent.context_limit_reached = False + agent.current_token_count = 0 + return True, pruned + return False, None + + if not result.compression_performed: + logger.warning("Compression not performed") + return False, None + + # Check if compression actually reduced tokens + if result.metadata: + if result.metadata.compressed_token_count >= result.metadata.original_token_count: + logger.warning( + "Compression did not reduce token count; falling back to minimal pruning" + ) + pruned = self._prune_messages_minimal(messages) + if pruned: + agent.context_limit_reached = False + agent.current_token_count = 0 + return True, pruned + return False, None + + logger.info( + f"Mid-execution compression successful - ratio: {result.metadata.compression_ratio:.1f}x, " + f"saved {result.metadata.original_token_count - result.metadata.compressed_token_count} tokens" + ) + + # Also store the compression summary as a visible message + if result.metadata: + conversation_service.append_compression_message( + agent.conversation_id, result.metadata.to_dict() + ) + + # Update agent's compressed summary for downstream persistence + agent.compressed_summary = result.compressed_summary + agent.compression_metadata = result.metadata.to_dict() if result.metadata else None + agent.compression_saved = False + + # Reset the context limit flag so tools can continue + agent.context_limit_reached = False + agent.current_token_count = 0 + + # Rebuild messages + rebuilt_messages = self._rebuild_messages_after_compression( + messages, + result.compressed_summary, + result.recent_queries, + include_current_execution=False, + include_tool_calls=False, + ) + + if rebuilt_messages is None: + return False, None + + return True, rebuilt_messages + + except Exception as e: + logger.error( + f"Error performing mid-execution compression: {str(e)}", exc_info=True + ) + return False, None + + def _perform_in_memory_compression( + self, agent, messages: List[Dict] + ) -> tuple[bool, Optional[List[Dict]]]: + """ + Fallback compression path when the conversation is not yet persisted. + + Uses CompressionService directly without DB persistence. + """ + try: + from application.api.answer.services.compression.service import ( + CompressionService, + ) + from application.core.model_utils import ( + get_api_key_for_provider, + get_provider_from_model_id, + ) + from application.core.settings import settings + from application.llm.llm_creator import LLMCreator + + conversation = self._build_conversation_from_messages(messages) + if not conversation: + logger.warning( + "Cannot perform in-memory compression: no user/assistant turns found" + ) + return False, None + + compression_model = ( + settings.COMPRESSION_MODEL_OVERRIDE + if settings.COMPRESSION_MODEL_OVERRIDE + else agent.model_id + ) + provider = get_provider_from_model_id(compression_model) + api_key = get_api_key_for_provider(provider) + compression_llm = LLMCreator.create_llm( + provider, + api_key, + getattr(agent, "user_api_key", None), + getattr(agent, "decoded_token", None), + model_id=compression_model, + ) + + # Create service without DB persistence capability + compression_service = CompressionService( + llm=compression_llm, + model_id=compression_model, + conversation_service=None, # No DB updates for in-memory + ) + + queries_count = len(conversation.get("queries", [])) + compress_up_to = queries_count - 1 + + if compress_up_to < 0 or queries_count == 0: + logger.warning("Not enough queries to compress in-memory context") + return False, None + + metadata = compression_service.compress_conversation( + conversation, + compress_up_to_index=compress_up_to, + ) + + # If compression doesn't reduce tokens, fall back to minimal pruning + if ( + metadata.compressed_token_count + >= metadata.original_token_count + ): + logger.warning( + "In-memory compression did not reduce token count; falling back to minimal pruning" + ) + pruned = self._prune_messages_minimal(messages) + if pruned: + agent.context_limit_reached = False + agent.current_token_count = 0 + return True, pruned + return False, None + + # Attach metadata to synthetic conversation + conversation["compression_metadata"] = { + "is_compressed": True, + "compression_points": [metadata.to_dict()], + } + + compressed_summary, recent_queries = ( + compression_service.get_compressed_context(conversation) + ) + + agent.compressed_summary = compressed_summary + agent.compression_metadata = metadata.to_dict() + agent.compression_saved = False + agent.context_limit_reached = False + agent.current_token_count = 0 + + rebuilt_messages = self._rebuild_messages_after_compression( + messages, + compressed_summary, + recent_queries, + include_current_execution=False, + include_tool_calls=False, + ) + if rebuilt_messages is None: + return False, None + + logger.info( + f"In-memory compression successful - ratio: {metadata.compression_ratio:.1f}x, " + f"saved {metadata.original_token_count - metadata.compressed_token_count} tokens" + ) + return True, rebuilt_messages + + except Exception as e: + logger.error( + f"Error performing in-memory compression: {str(e)}", exc_info=True + ) + return False, None + def handle_tool_calls( self, agent, tool_calls: List[ToolCall], tools_dict: Dict, messages: List[Dict] ) -> Generator: @@ -195,7 +597,110 @@ class LLMHandler(ABC): """ updated_messages = messages.copy() - for call in tool_calls: + for i, call in enumerate(tool_calls): + # Check context limit before executing tool call + if hasattr(agent, '_check_context_limit') and agent._check_context_limit(updated_messages): + # Context limit reached - attempt mid-execution compression + compression_attempted = False + compression_successful = False + + try: + from application.core.settings import settings + compression_enabled = settings.ENABLE_CONVERSATION_COMPRESSION + except Exception: + compression_enabled = False + + if compression_enabled: + compression_attempted = True + try: + logger.info( + f"Context limit reached with {len(tool_calls) - i} remaining tool calls. " + f"Attempting mid-execution compression..." + ) + + # Trigger mid-execution compression (DB-backed if available, otherwise in-memory) + compression_successful, rebuilt_messages = self._perform_mid_execution_compression( + agent, updated_messages + ) + + if compression_successful and rebuilt_messages is not None: + # Update the messages list with rebuilt compressed version + updated_messages = rebuilt_messages + + # Yield compression success message + yield { + "type": "info", + "data": { + "message": "Context window limit reached. Compressed conversation history to continue processing." + } + } + + logger.info( + f"Mid-execution compression successful. Continuing with {len(tool_calls) - i} remaining tool calls." + ) + # Proceed to execute the current tool call with the reduced context + else: + logger.warning("Mid-execution compression attempted but failed. Skipping remaining tools.") + except Exception as e: + logger.error(f"Error during mid-execution compression: {str(e)}", exc_info=True) + compression_attempted = True + compression_successful = False + + # If compression wasn't attempted or failed, skip remaining tools + if not compression_successful: + if i == 0: + # Special case: limit reached before executing any tools + # This can happen when previous tool responses pushed context over limit + if compression_attempted: + logger.warning( + f"Context limit reached before executing any tools. " + f"Compression attempted but failed. " + f"Skipping all {len(tool_calls)} pending tool call(s). " + f"This typically occurs when previous tool responses contained large amounts of data." + ) + else: + logger.warning( + f"Context limit reached before executing any tools. " + f"Skipping all {len(tool_calls)} pending tool call(s). " + f"This typically occurs when previous tool responses contained large amounts of data. " + f"Consider enabling compression or using a model with larger context window." + ) + else: + # Normal case: executed some tools, now stopping + tool_word = "tool call" if i == 1 else "tool calls" + remaining = len(tool_calls) - i + remaining_word = "tool call" if remaining == 1 else "tool calls" + if compression_attempted: + logger.warning( + f"Context limit reached after executing {i} {tool_word}. " + f"Compression attempted but failed. " + f"Skipping remaining {remaining} {remaining_word}." + ) + else: + logger.warning( + f"Context limit reached after executing {i} {tool_word}. " + f"Skipping remaining {remaining} {remaining_word}. " + f"Consider enabling compression or using a model with larger context window." + ) + + # Mark remaining tools as skipped + for remaining_call in tool_calls[i:]: + skip_message = { + "type": "tool_call", + "data": { + "tool_name": "system", + "call_id": remaining_call.id, + "action_name": remaining_call.name, + "arguments": {}, + "result": "Skipped: Context limit reached. Too many tool calls in conversation.", + "status": "skipped" + } + } + yield skip_message + + # Set flag on agent + agent.context_limit_reached = True + break try: self.tool_calls.append(call) tool_executor_gen = agent._execute_tool_action(tools_dict, call) @@ -205,21 +710,26 @@ class LLMHandler(ABC): except StopIteration as e: tool_response, call_id = e.value break + + function_call_content = { + "function_call": { + "name": call.name, + "args": call.arguments, + "call_id": call_id, + } + } + # Include thought_signature for Google Gemini 3 models + # It should be at the same level as function_call, not inside it + if call.thought_signature: + function_call_content["thought_signature"] = call.thought_signature updated_messages.append( { "role": "assistant", - "content": [ - { - "function_call": { - "name": call.name, - "args": call.arguments, - "call_id": call_id, - } - } - ], + "content": [function_call_content], } ) + updated_messages.append(self.create_tool_message(call, tool_response)) except Exception as e: logger.error(f"Error executing tool: {str(e)}", exc_info=True) @@ -324,6 +834,9 @@ class LLMHandler(ABC): existing.name = call.name if call.arguments: existing.arguments += call.arguments + # Preserve thought_signature for Google Gemini 3 models + if call.thought_signature: + existing.thought_signature = call.thought_signature if parsed.finish_reason == "tool_calls": tool_handler_gen = self.handle_tool_calls( agent, list(tool_calls.values()), tools_dict, messages @@ -336,8 +849,21 @@ class LLMHandler(ABC): break tool_calls = {} + # Check if context limit was reached during tool execution + if hasattr(agent, 'context_limit_reached') and agent.context_limit_reached: + # Add system message warning about context limit + messages.append({ + "role": "system", + "content": ( + "WARNING: Context window limit has been reached. " + "Please provide a final response to the user without making additional tool calls. " + "Summarize the work completed so far." + ) + }) + logger.info("Context limit reached - instructing agent to wrap up") + response = agent.llm.gen_stream( - model=agent.model_id, messages=messages, tools=agent.tools + model=agent.model_id, messages=messages, tools=agent.tools if not agent.context_limit_reached else None ) self.llm_calls.append(build_stack_data(agent.llm)) diff --git a/application/llm/handlers/google.py b/application/llm/handlers/google.py index 7fa44cb6..0142922a 100644 --- a/application/llm/handlers/google.py +++ b/application/llm/handlers/google.py @@ -19,15 +19,20 @@ class GoogleLLMHandler(LLMHandler): ) if hasattr(response, "candidates"): parts = response.candidates[0].content.parts if response.candidates else [] - tool_calls = [ - ToolCall( - id=str(uuid.uuid4()), - name=part.function_call.name, - arguments=part.function_call.args, - ) - for part in parts - if hasattr(part, "function_call") and part.function_call is not None - ] + tool_calls = [] + for idx, part in enumerate(parts): + if hasattr(part, "function_call") and part.function_call is not None: + has_sig = hasattr(part, "thought_signature") and part.thought_signature is not None + thought_sig = part.thought_signature if has_sig else None + tool_calls.append( + ToolCall( + id=str(uuid.uuid4()), + name=part.function_call.name, + arguments=part.function_call.args, + index=idx, + thought_signature=thought_sig, + ) + ) content = " ".join( part.text @@ -41,13 +46,17 @@ class GoogleLLMHandler(LLMHandler): raw_response=response, ) else: + # This branch handles individual Part objects from streaming responses tool_calls = [] - if hasattr(response, "function_call"): + if hasattr(response, "function_call") and response.function_call is not None: + has_sig = hasattr(response, "thought_signature") and response.thought_signature is not None + thought_sig = response.thought_signature if has_sig else None tool_calls.append( ToolCall( id=str(uuid.uuid4()), name=response.function_call.name, arguments=response.function_call.args, + thought_signature=thought_sig, ) ) return LLMResponse( diff --git a/application/llm/openai.py b/application/llm/openai.py index beab465b..3917cbf7 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -128,6 +128,10 @@ class OpenAILLM(BaseLLM): ): messages = self._clean_messages_openai(messages) + # Convert max_tokens to max_completion_tokens for newer models + if "max_tokens" in kwargs: + kwargs["max_completion_tokens"] = kwargs.pop("max_tokens") + request_params = { "model": model, "messages": messages, @@ -159,6 +163,10 @@ class OpenAILLM(BaseLLM): ): messages = self._clean_messages_openai(messages) + # Convert max_tokens to max_completion_tokens for newer models + if "max_tokens" in kwargs: + kwargs["max_completion_tokens"] = kwargs.pop("max_tokens") + request_params = { "model": model, "messages": messages, diff --git a/application/prompts/compression/v1.0.txt b/application/prompts/compression/v1.0.txt new file mode 100644 index 00000000..28e7550d --- /dev/null +++ b/application/prompts/compression/v1.0.txt @@ -0,0 +1,35 @@ +Your task is to create a detailed summary of the conversation so far, paying close attention to the user's explicit requests and your previous actions. + +This summary should be thorough in capturing technical details, code patterns, and architectural decisions that would be essential for continuing work without losing context. + +Before providing your final summary, wrap your analysis in tags to organize your thoughts and ensure you've covered all necessary points. In your analysis process: + +1. Chronologically analyze each message, tool call and section of the conversation. For each section thoroughly identify: + - The user's explicit requests and intents + - Your approach to addressing the user's requests + - Key decisions, concepts and patterns + - Specific details like if applicable: + - file names + - full code snippets + - function signatures + - file edits + - Errors that you ran into and how you fixed them + - Pay special attention to specific user feedback that you received, especially if the user told you to do something differently. + +2. Double-check for accuracy and completeness, addressing each required element thoroughly. + +Your summary should include the following sections: + +1. Primary Request and Intent: Capture all of the user's explicit requests and intents in detail +2. Key Concepts: List all important concepts discussed. +3. Files and Code Sections: Enumerate specific files and code sections examined, modified, or created. Pay special attention to the most recent messages and include full code snippets where applicable and include a summary of why this file read or edit is important. +4. Errors and fixes: List all errors that you ran into, and how you fixed them. Pay special attention to specific user feedback that you received, especially if the user told you to do something differently. +5. Problem Solving: Document problems solved and any ongoing troubleshooting efforts. +6. All user messages: List ALL user messages that are not tool results. These are critical for understanding the users' feedback and changing intent. +7. Tool Calls: List ALL tool calls made, including their inputs relevant parts of the outputs. +8. Pending Tasks: Outline any pending tasks that you have explicitly been asked to work on. +9. Current Work: Describe in detail precisely what was being worked on immediately before this summary request, paying special attention to the most recent messages from both user and assistant. Include file names and code snippets where applicable. +10. Optional Next Step: List the next step that you will take that is related to the most recent work you were doing. IMPORTANT: ensure that this step is DIRECTLY in line with the user's most recent explicit requests, and the task you were working on immediately before this summary request. If your last task was concluded, then only list next steps if they are explicitly in line with the users request. Do not start on tangential requests or really old requests that were already completed without confirming with the user first. +If there is a next step, include direct quotes from the most recent conversation showing exactly what task you were working on and where you left off. This should be verbatim to ensure there's no drift in task interpretation. + +Please provide your summary based on the conversation and tools used so far, following this structure and ensuring precision and thoroughness in your response. diff --git a/application/requirements.txt b/application/requirements.txt index 08d259b1..cb58247b 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -15,7 +15,7 @@ Flask==3.1.1 faiss-cpu==1.9.0.post1 fastmcp==2.11.0 flask-restx==1.3.0 -google-genai==1.3.0 +google-genai==1.49.0 google-api-python-client==2.179.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.2 diff --git a/application/utils.py b/application/utils.py index 89b884f0..b25c4717 100644 --- a/application/utils.py +++ b/application/utils.py @@ -197,6 +197,24 @@ def generate_image_url(image_path): return f"{base_url}/api/images/{image_path}" +def calculate_compression_threshold( + model_id: str, threshold_percentage: float = 0.8 +) -> int: + """ + Calculate token threshold for triggering compression. + + Args: + model_id: Model identifier + threshold_percentage: Percentage of context window (default 80%) + + Returns: + Token count threshold + """ + total_context = get_token_limit(model_id) + threshold = int(total_context * threshold_percentage) + return threshold + + def clean_text_for_tts(text: str) -> str: """ clean text for Text-to-Speech processing. diff --git a/tests/llm/test_google_llm.py b/tests/llm/test_google_llm.py index 0862c727..80434e98 100644 --- a/tests/llm/test_google_llm.py +++ b/tests/llm/test_google_llm.py @@ -91,7 +91,7 @@ def test_clean_messages_google_basic(): {"function_call": {"name": "fn", "args": {"a": 1}}}, ]}, ] - cleaned = llm._clean_messages_google(msgs) + cleaned, system_instruction = llm._clean_messages_google(msgs) assert all(hasattr(c, "role") and hasattr(c, "parts") for c in cleaned) assert any(c.role == "model" for c in cleaned) diff --git a/tests/test_agent_token_tracking.py b/tests/test_agent_token_tracking.py new file mode 100644 index 00000000..e168567a --- /dev/null +++ b/tests/test_agent_token_tracking.py @@ -0,0 +1,325 @@ +import pytest +from unittest.mock import Mock, patch + +from application.agents.base import BaseAgent +from application.llm.handlers.base import LLMHandler, ToolCall + + +class MockAgent(BaseAgent): + """Mock agent for testing""" + + def _gen_inner(self, query, log_context=None): + yield {"answer": "test"} + + +@pytest.fixture +def mock_agent(): + """Create a mock agent for testing""" + agent = MockAgent( + endpoint="test", + llm_name="openai", + model_id="gpt-4o", + api_key="test-key", + ) + agent.llm = Mock() + return agent + + +@pytest.fixture +def mock_llm_handler(): + """Create a mock LLM handler""" + handler = Mock(spec=LLMHandler) + handler.tool_calls = [] + return handler + + +class TestAgentTokenTracking: + """Test suite for agent token tracking during execution""" + + def test_calculate_current_context_tokens(self, mock_agent): + """Test token calculation for current context""" + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello, how are you?"}, + {"role": "assistant", "content": "I'm doing well, thank you!"}, + ] + + tokens = mock_agent._calculate_current_context_tokens(messages) + + # Should count tokens from all messages + assert tokens > 0 + # Rough estimate: ~20-40 tokens for this conversation + assert 15 < tokens < 60 + + def test_calculate_tokens_with_tool_calls(self, mock_agent): + """Test token calculation includes tool call content""" + messages = [ + {"role": "system", "content": "Test"}, + { + "role": "assistant", + "content": [ + { + "function_call": { + "name": "search_tool", + "args": {"query": "test"}, + "call_id": "123", + } + } + ], + }, + { + "role": "tool", + "content": [ + { + "function_response": { + "name": "search_tool", + "response": {"result": "Found 10 results"}, + "call_id": "123", + } + } + ], + }, + ] + + tokens = mock_agent._calculate_current_context_tokens(messages) + + # Should include tool call tokens + assert tokens > 0 + + @patch("application.core.model_utils.get_token_limit") + @patch("application.core.settings.settings") + def test_check_context_limit_below_threshold( + self, mock_settings, mock_get_token_limit, mock_agent + ): + """Test context limit check when below threshold""" + mock_get_token_limit.return_value = 128000 + mock_settings.COMPRESSION_THRESHOLD_PERCENTAGE = 0.8 + + messages = [ + {"role": "system", "content": "Short message"}, + {"role": "user", "content": "Hello"}, + ] + + # Should return False for small conversation + result = mock_agent._check_context_limit(messages) + assert result is False + + # Should track current token count + assert mock_agent.current_token_count > 0 + assert mock_agent.current_token_count < 128000 * 0.8 + + @patch("application.core.model_utils.get_token_limit") + @patch("application.core.settings.settings") + def test_check_context_limit_above_threshold( + self, mock_settings, mock_get_token_limit, mock_agent + ): + """Test context limit check when above threshold""" + mock_get_token_limit.return_value = 100 # Very small limit for testing + mock_settings.COMPRESSION_THRESHOLD_PERCENTAGE = 0.8 + + # Create messages that will exceed 80 tokens (80% of 100) + messages = [ + {"role": "system", "content": "a " * 50}, # ~50 tokens + {"role": "user", "content": "b " * 50}, # ~50 tokens + ] + + # Should return True when exceeding threshold + result = mock_agent._check_context_limit(messages) + assert result is True + + @patch("application.agents.base.logger") + def test_check_context_limit_error_handling(self, mock_logger, mock_agent): + """Test error handling in context limit check""" + # Force an error by making get_token_limit fail + with patch( + "application.core.model_utils.get_token_limit", side_effect=Exception("Test error") + ): + messages = [{"role": "user", "content": "test"}] + + result = mock_agent._check_context_limit(messages) + + # Should return False on error (safe default) + assert result is False + # Should log the error + assert mock_logger.error.called + + def test_context_limit_flag_initialization(self, mock_agent): + """Test that context limit flag is initialized""" + assert hasattr(mock_agent, "context_limit_reached") + assert mock_agent.context_limit_reached is False + + assert hasattr(mock_agent, "current_token_count") + assert mock_agent.current_token_count == 0 + + +class TestLLMHandlerTokenTracking: + """Test suite for LLM handler token tracking""" + + @patch("application.llm.handlers.base.logger") + def test_handle_tool_calls_stops_at_limit(self, mock_logger): + """Test that tool execution stops when context limit is reached""" + from application.llm.handlers.base import LLMHandler + + # Create a concrete handler for testing + class TestHandler(LLMHandler): + def parse_response(self, response): + pass + + def create_tool_message(self, tool_call, result): + return {"role": "tool", "content": str(result)} + + def _iterate_stream(self, response): + yield "" + + handler = TestHandler() + + # Create mock agent that hits limit on second tool + mock_agent = Mock() + mock_agent.context_limit_reached = False + + call_count = [0] + + def check_limit_side_effect(messages): + call_count[0] += 1 + # Return True on second call (second tool) + return call_count[0] >= 2 + + mock_agent._check_context_limit = Mock(side_effect=check_limit_side_effect) + mock_agent._execute_tool_action = Mock( + return_value=iter([{"type": "tool_call", "data": {}}]) + ) + + # Create multiple tool calls + tool_calls = [ + ToolCall(id="1", name="tool1", arguments={}), + ToolCall(id="2", name="tool2", arguments={}), + ToolCall(id="3", name="tool3", arguments={}), + ] + + messages = [] + tools_dict = {} + + # Execute tool calls + results = list(handler.handle_tool_calls(mock_agent, tool_calls, tools_dict, messages)) + + # First tool should execute + assert mock_agent._execute_tool_action.call_count == 1 + + # Should have yielded skip messages for tools 2 and 3 + skip_messages = [r for r in results if r.get("type") == "tool_call" and r.get("data", {}).get("status") == "skipped"] + assert len(skip_messages) == 2 + + # Should have set the flag + assert mock_agent.context_limit_reached is True + + # Should have logged warning + assert mock_logger.warning.called + + def test_handle_tool_calls_all_execute_when_no_limit(self): + """Test that all tools execute when under limit""" + from application.llm.handlers.base import LLMHandler + + class TestHandler(LLMHandler): + def parse_response(self, response): + pass + + def create_tool_message(self, tool_call, result): + return {"role": "tool", "content": str(result)} + + def _iterate_stream(self, response): + yield "" + + handler = TestHandler() + + # Create mock agent that never hits limit + mock_agent = Mock() + mock_agent.context_limit_reached = False + mock_agent._check_context_limit = Mock(return_value=False) + mock_agent._execute_tool_action = Mock( + return_value=iter([{"type": "tool_call", "data": {}}]) + ) + + tool_calls = [ + ToolCall(id="1", name="tool1", arguments={}), + ToolCall(id="2", name="tool2", arguments={}), + ToolCall(id="3", name="tool3", arguments={}), + ] + + messages = [] + tools_dict = {} + + # Execute tool calls + list(handler.handle_tool_calls(mock_agent, tool_calls, tools_dict, messages)) + + # All 3 tools should execute + assert mock_agent._execute_tool_action.call_count == 3 + + # Should not have set the flag + assert mock_agent.context_limit_reached is False + + @patch("application.llm.handlers.base.logger") + def test_handle_streaming_adds_warning_message(self, mock_logger): + """Test that streaming handler adds warning when limit reached""" + from application.llm.handlers.base import LLMHandler, LLMResponse, ToolCall + + class TestHandler(LLMHandler): + def parse_response(self, response): + if isinstance(response, dict) and response.get("type") == "tool_call": + return LLMResponse( + content="", + tool_calls=[ToolCall(id="1", name="test", arguments={}, index=0)], + finish_reason="tool_calls", + raw_response=None, + ) + else: + return LLMResponse( + content="Done", + tool_calls=[], + finish_reason="stop", + raw_response=None, + ) + + def create_tool_message(self, tool_call, result): + return {"role": "tool", "content": str(result)} + + def _iterate_stream(self, response): + if response == "first": + yield {"type": "tool_call"} # Object to be parsed, not string + else: + yield {"type": "stop"} # Object to be parsed, not string + + handler = TestHandler() + + # Create mock agent with limit reached + mock_agent = Mock() + mock_agent.context_limit_reached = True + mock_agent.model_id = "gpt-4o" + mock_agent.tools = [] + mock_agent.llm = Mock() + mock_agent.llm.gen_stream = Mock(return_value="second") + + def tool_handler_gen(*args): + yield {"type": "tool", "data": {}} + return [] + + # Mock handle_tool_calls to return messages and set flag + with patch.object( + handler, "handle_tool_calls", return_value=tool_handler_gen() + ): + messages = [] + tools_dict = {} + + # Execute streaming + list(handler.handle_streaming(mock_agent, "first", tools_dict, messages)) + + # Should have called gen_stream with tools=None (disabled) + mock_agent.llm.gen_stream.assert_called() + call_kwargs = mock_agent.llm.gen_stream.call_args.kwargs + assert call_kwargs.get("tools") is None + + # Should have logged the warning + assert mock_logger.info.called + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_compression_service.py b/tests/test_compression_service.py new file mode 100644 index 00000000..c6700e3a --- /dev/null +++ b/tests/test_compression_service.py @@ -0,0 +1,1082 @@ +import pytest +from datetime import datetime, timezone +from unittest.mock import Mock, patch + +from application.api.answer.services.compression import CompressionService +from application.api.answer.services.compression.threshold_checker import ( + CompressionThresholdChecker, +) +from application.api.answer.services.compression.token_counter import TokenCounter +from application.api.answer.services.compression.prompt_builder import ( + CompressionPromptBuilder, +) +from application.core.settings import settings + + +@pytest.fixture +def mock_llm(): + """Create a mock LLM for testing""" + llm = Mock() + llm.gen = Mock() + return llm + + +@pytest.fixture +def compression_service(mock_llm): + """Create a CompressionService instance with mock LLM""" + return CompressionService(llm=mock_llm, model_id="gpt-4o") + + +@pytest.fixture +def threshold_checker(): + """Create a ThresholdChecker instance""" + return CompressionThresholdChecker() + + +@pytest.fixture +def prompt_builder(): + """Create a PromptBuilder instance""" + return CompressionPromptBuilder() + + +@pytest.fixture +def sample_conversation(): + """Create a sample conversation for testing""" + return { + "_id": "test_conversation_id", + "user": "test_user", + "date": datetime.now(timezone.utc), + "name": "Test Conversation", + "queries": [ + { + "prompt": "What is Python?", + "response": "Python is a high-level programming language.", + "thought": "", + "sources": [], + "tool_calls": [], + "timestamp": datetime.now(timezone.utc), + }, + { + "prompt": "How do I install it?", + "response": "You can install Python from python.org", + "thought": "", + "sources": [], + "tool_calls": [], + "timestamp": datetime.now(timezone.utc), + }, + { + "prompt": "What are some popular libraries?", + "response": "Popular Python libraries include NumPy, Pandas, Django, Flask, etc.", + "thought": "", + "sources": [], + "tool_calls": [], + "timestamp": datetime.now(timezone.utc), + }, + ], + } + + +@pytest.fixture +def large_conversation(): + """Create a large conversation that exceeds threshold""" + queries = [] + for i in range(100): + queries.append( + { + "prompt": f"Question {i}: " + ("test " * 100), # ~400 tokens each + "response": f"Answer {i}: " + ("response " * 100), # ~400 tokens each + "thought": "", + "sources": [], + "tool_calls": [], + "timestamp": datetime.now(timezone.utc), + } + ) + + return { + "_id": "large_conversation_id", + "user": "test_user", + "date": datetime.now(timezone.utc), + "name": "Large Conversation", + "queries": queries, + } + + +class TestCompressionService: + """Test suite for CompressionService""" + + def test_initialization(self, mock_llm): + """Test CompressionService initialization""" + service = CompressionService(llm=mock_llm, model_id="gpt-4o") + + assert service.llm == mock_llm + assert service.model_id == "gpt-4o" + assert service.prompt_builder is not None + assert service.prompt_builder.version == settings.COMPRESSION_PROMPT_VERSION + + @patch("application.api.answer.services.compression.threshold_checker.get_token_limit") + def test_should_compress_below_threshold( + self, mock_get_token_limit, threshold_checker, sample_conversation + ): + """Test that compression is not triggered when below threshold""" + mock_get_token_limit.return_value = 128000 # GPT-4o limit + + # Small conversation should not trigger compression + result = threshold_checker.should_compress( + sample_conversation, model_id="gpt-4o" + ) + + assert result is False + + @patch("application.api.answer.services.compression.threshold_checker.get_token_limit") + def test_should_compress_above_threshold( + self, mock_get_token_limit, threshold_checker, large_conversation + ): + """Test that compression is triggered when above threshold""" + mock_get_token_limit.return_value = 10000 # Lower limit to ensure large conversation exceeds threshold + + # Large conversation should trigger compression (100 queries with repeated text) + # Threshold at 80% of 10k = 8k tokens, so large_conversation > 8k should trigger + result = threshold_checker.should_compress( + large_conversation, model_id="gpt-4o" + ) + + assert result is True + + @patch("application.api.answer.services.compression.threshold_checker.get_token_limit") + def test_should_compress_at_exact_threshold( + self, mock_get_token_limit, threshold_checker + ): + """Test compression trigger at exact 80% threshold""" + mock_get_token_limit.return_value = 1000 + + # Create conversation with exactly 800 tokens (80% of 1000) + conversation = { + "queries": [ + { + "prompt": "a " * 200, # ~200 tokens + "response": "b " * 200, # ~200 tokens + }, + { + "prompt": "c " * 200, # ~200 tokens + "response": "d " * 200, # ~200 tokens + }, + ] + } + + result = threshold_checker.should_compress(conversation, model_id="test-model") + + # Should trigger at or above 80% + assert result is True + + def test_compress_conversation_basic(self, compression_service, sample_conversation): + """Test basic conversation compression""" + # Mock LLM response + mock_summary = """ + + The conversation covers Python basics and installation. + + + + 1. Primary Request and Intent: + User asked about Python and how to install it. + + 2. Key Concepts: + - Python programming language + - Installation process + + 3. Files and Code Sections: + None + + 4. Errors and fixes: + None + + 5. Problem Solving: + Explained Python installation from python.org + + 6. All user messages: + - What is Python? + - How do I install it? + - What are some popular libraries? + + 7. Pending Tasks: + None + + 8. Current Work: + Provided information about popular Python libraries. + + 9. Optional Next Step: + None + + """ + compression_service.llm.gen.return_value = mock_summary + + # Compress first 2 queries + result = compression_service.compress_conversation( + conversation=sample_conversation, compress_up_to_index=1 + ) + + # Verify LLM was called + assert compression_service.llm.gen.called + + # Verify result is a CompressionMetadata object + assert hasattr(result, 'timestamp') + assert result.query_index == 1 + assert hasattr(result, 'compressed_summary') + assert result.original_token_count > 0 + assert result.compressed_token_count > 0 + assert result.compression_ratio > 0 + assert result.model_used == "gpt-4o" + assert result.compression_prompt_version == settings.COMPRESSION_PROMPT_VERSION + + # Verify summary was extracted correctly (without analysis tags) + assert "" not in result.compressed_summary + assert "Primary Request and Intent" in result.compressed_summary + + def test_compress_conversation_with_tool_calls(self, compression_service): + """Test compression of conversation with tool calls""" + conversation = { + "queries": [ + { + "prompt": "Search for Python tutorials", + "response": "I'll search for Python tutorials.", + "thought": "Need to use search tool", + "sources": [], + "tool_calls": [ + { + "tool_name": "search_tool", + "action_name": "search", + "arguments": {"query": "Python tutorials"}, + "result": "Found 100 tutorials", + "status": "completed", + } + ], + "timestamp": datetime.now(timezone.utc), + } + ] + } + + mock_summary = "Test summary with tools" + compression_service.llm.gen.return_value = mock_summary + + compression_service.compress_conversation( + conversation=conversation, compress_up_to_index=0 + ) + + # Verify tool calls are included in compression prompt + call_args = compression_service.llm.gen.call_args + messages = call_args[1]["messages"] + user_message = messages[1]["content"] + + assert "Tool Calls:" in user_message + assert "search_tool" in user_message + + def test_compress_conversation_invalid_index( + self, compression_service, sample_conversation + ): + """Test compression with invalid index raises error""" + with pytest.raises(ValueError, match="Invalid compress_up_to_index"): + compression_service.compress_conversation( + conversation=sample_conversation, + compress_up_to_index=100, # Invalid - conversation only has 3 queries + ) + + def test_get_compressed_context_no_compression( + self, compression_service, sample_conversation + ): + """Test getting context when no compression exists""" + summary, recent = compression_service.get_compressed_context( + sample_conversation + ) + + assert summary is None + assert len(recent) == 3 # All queries returned + + def test_get_compressed_context_with_compression(self, compression_service): + """Test getting context when compression exists""" + conversation = { + "queries": [ + {"prompt": "Q1", "response": "A1"}, + {"prompt": "Q2", "response": "A2"}, + {"prompt": "Q3", "response": "A3"}, + {"prompt": "Q4", "response": "A4"}, + {"prompt": "Q5", "response": "A5"}, + ], + "compression_metadata": { + "is_compressed": True, + "last_compression_at": datetime.now(timezone.utc), + "compression_points": [ + { + "timestamp": datetime.now(timezone.utc), + "query_index": 2, # Compressed up to Q3 + "compressed_summary": "Summary of Q1-Q3", + "original_token_count": 100, + "compressed_token_count": 20, + "compression_ratio": 5.0, + } + ], + }, + } + + summary, recent = compression_service.get_compressed_context( + conversation + ) + + assert summary == "Summary of Q1-Q3" + assert len(recent) == 2 # Q4 and Q5 (after compression point) + assert recent[0]["prompt"] == "Q4" + assert recent[1]["prompt"] == "Q5" + + def test_get_compressed_context_multiple_compressions(self, compression_service): + """Test getting context when multiple compressions exist""" + conversation = { + "queries": [ + {"prompt": f"Q{i}", "response": f"A{i}"} for i in range(1, 11) + ], + "compression_metadata": { + "is_compressed": True, + "last_compression_at": datetime.now(timezone.utc), + "compression_points": [ + { + "timestamp": datetime.now(timezone.utc), + "query_index": 4, # First compression + "compressed_summary": "First compression summary", + "original_token_count": 100, + "compressed_token_count": 20, + }, + { + "timestamp": datetime.now(timezone.utc), + "query_index": 7, # Second compression + "compressed_summary": "Second compression summary (includes first)", + "original_token_count": 150, + "compressed_token_count": 30, + }, + ], + }, + } + + summary, recent = compression_service.get_compressed_context( + conversation + ) + + # Should use the most recent compression + assert summary == "Second compression summary (includes first)" + assert len(recent) == 2 # Q9 and Q10 (after compression point at index 7) + assert recent[0]["prompt"] == "Q9" + assert recent[1]["prompt"] == "Q10" + + def test_extract_summary_with_tags(self, compression_service): + """Test summary extraction with analysis and summary tags""" + llm_response = """ + + This is my analysis of the conversation. + It has multiple lines. + + + + This is the actual summary. + It should be extracted. + + """ + + result = compression_service._extract_summary(llm_response) + + assert "" not in result + assert "This is the actual summary" in result + assert "my analysis" not in result + + def test_extract_summary_without_tags(self, compression_service): + """Test summary extraction when no tags present""" + llm_response = "This is a plain summary without tags." + + result = compression_service._extract_summary(llm_response) + + assert result == "This is a plain summary without tags." + + def test_count_tokens_in_queries(self, sample_conversation): + """Test token counting in queries""" + queries = sample_conversation["queries"] + + token_count = TokenCounter.count_query_tokens(queries) + + # Should count all prompts and responses + assert token_count > 0 + + def test_count_tokens_with_tool_calls(self): + """Test token counting includes tool calls""" + queries = [ + { + "prompt": "Test prompt", + "response": "Test response", + "tool_calls": [ + { + "tool_name": "test_tool", + "action_name": "test_action", + "arguments": {"arg": "value"}, + "result": "Tool result", + } + ], + } + ] + + token_count_with_tools = TokenCounter.count_query_tokens( + queries, include_tool_calls=True + ) + token_count_without_tools = TokenCounter.count_query_tokens( + queries, include_tool_calls=False + ) + + assert token_count_with_tools > token_count_without_tools + + def test_format_conversation_for_compression( + self, prompt_builder, sample_conversation + ): + """Test conversation formatting for compression prompt""" + queries = sample_conversation["queries"] + + formatted = prompt_builder._format_conversation(queries) + + # Verify formatting includes all messages + assert "Message 1" in formatted + assert "What is Python?" in formatted + assert "Python is a high-level programming language" in formatted + assert "Message 2" in formatted + assert "How do I install it?" in formatted + + def test_build_compression_prompt_basic(self, prompt_builder): + """Test compression prompt building""" + queries = [ + {"prompt": "Q1", "response": "A1", "tool_calls": [], "sources": []}, + {"prompt": "Q2", "response": "A2", "tool_calls": [], "sources": []}, + ] + + messages = prompt_builder.build_prompt(queries) + + assert len(messages) == 2 # System and user messages + assert messages[0]["role"] == "system" + assert messages[1]["role"] == "user" + assert "conversation to summarize" in messages[1]["content"] + + def test_build_compression_prompt_with_existing_compressions( + self, prompt_builder + ): + """Test compression prompt building with existing compressions""" + queries = [ + {"prompt": "Q3", "response": "A3", "tool_calls": [], "sources": []}, + {"prompt": "Q4", "response": "A4", "tool_calls": [], "sources": []}, + ] + + existing_compressions = [ + { + "query_index": 1, + "compressed_summary": "Previous compression summary", + "timestamp": datetime.now(timezone.utc), + } + ] + + messages = prompt_builder.build_prompt( + queries, existing_compressions + ) + + user_content = messages[1]["content"] + + # Should mention existing compression + assert "compressed before" in user_content + assert "Previous compression summary" in user_content + assert "NEW summary" in user_content + + def test_calculate_conversation_tokens( + self, sample_conversation + ): + """Test conversation token calculation""" + token_count = TokenCounter.count_conversation_tokens( + sample_conversation, include_system_prompt=False + ) + + assert token_count > 0 + + # With system prompt should be higher + token_count_with_system = TokenCounter.count_conversation_tokens( + sample_conversation, include_system_prompt=True + ) + + assert token_count_with_system > token_count + + @patch("application.api.answer.services.compression.threshold_checker.logger") + def test_error_handling_in_should_compress( + self, mock_logger, threshold_checker, sample_conversation + ): + """Test error handling in should_compress""" + # Force an error by making get_token_limit raise an exception + with patch( + "application.api.answer.services.compression.threshold_checker.get_token_limit", + side_effect=Exception("Test error"), + ): + result = threshold_checker.should_compress( + sample_conversation, model_id="gpt-4o" + ) + + # Should return False on error + assert result is False + # Should log the error + assert mock_logger.error.called + + @patch("application.api.answer.services.compression.service.logger") + def test_error_handling_in_get_compressed_context( + self, mock_logger, compression_service + ): + """Test error handling in get_compressed_context""" + # Malformed conversation + malformed_conversation = {"queries": None} + + summary, recent = compression_service.get_compressed_context( + malformed_conversation + ) + + # Should return safe defaults + assert summary is None + assert recent == [] + # Should log the error + assert mock_logger.error.called + + + def test_compression_points_array_limiting(self, compression_service): + """Test that only the most recent compression points are kept""" + # Simulate a conversation with 3 previous compressions + conversation = { + "queries": [ + {"prompt": f"Q{i}", "response": f"A{i}"} for i in range(1, 11) + ], + "compression_metadata": { + "is_compressed": True, + "last_compression_at": datetime.now(timezone.utc), + "compression_points": [ + { + "timestamp": datetime.now(timezone.utc), + "query_index": 2, + "compressed_summary": "First compression summary", + "original_token_count": 100, + "compressed_token_count": 20, + }, + { + "timestamp": datetime.now(timezone.utc), + "query_index": 5, + "compressed_summary": "Second compression summary", + "original_token_count": 150, + "compressed_token_count": 30, + }, + { + "timestamp": datetime.now(timezone.utc), + "query_index": 7, + "compressed_summary": "Third compression summary", + "original_token_count": 200, + "compressed_token_count": 40, + }, + ], + }, + } + + # The service should use the most recent compression + summary, recent = compression_service.get_compressed_context( + conversation + ) + + # Should use the most recent (third) compression + assert summary == "Third compression summary" + assert len(recent) == 2 # Q9 and Q10 (after compression point at index 7) + assert recent[0]["prompt"] == "Q9" + assert recent[1]["prompt"] == "Q10" + + def test_compression_with_heavy_tool_usage(self, compression_service): + """Test compression when conversation has many tool calls with large responses + + Scenario: User asks agent to scrape all files in a GitHub repo, generating + dozens of tool calls with file contents as responses. This tests the system's + ability to compress tool-heavy conversations that hit token limits. + """ + # Simulate a conversation where agent scraped 50 files from DocsGPT repo + queries = [] + + # Initial user request + queries.append({ + "prompt": "Please analyze all Python files in the https://github.com/arc53/DocsGPT repository", + "response": "I'll scrape all the Python files from the DocsGPT repository and analyze them.", + "tool_calls": [] + }) + + # Simulate 50 file scraping tool calls with realistic file contents + file_paths = [ + "application/app.py", + "application/api/answer/routes.py", + "application/api/answer/services/conversation_service.py", + "application/api/answer/services/compression_service.py", + "application/api/answer/services/stream_processor.py", + "application/agents/base.py", + "application/agents/react.py", + "application/llm/handlers/base.py", + "application/llm/llm_creator.py", + "application/core/settings.py", + "application/core/model_configs.py", + "application/utils.py", + "application/vectorstore/base.py", + "application/parser/file_parser.py", + "tests/test_compression_service.py", + "tests/test_agent_token_tracking.py", + "frontend/src/App.tsx", + "frontend/src/store/index.ts", + "deployment/docker-compose.yaml", + "setup.py", + ] + + tool_calls = [] + for i, file_path in enumerate(file_paths[:20]): # First 20 files + # Each tool call with realistic file content (simulating ~500-1000 tokens per file) + file_content = f""" +# {file_path} + +import os +import sys +from typing import Dict, List, Optional, Any +from datetime import datetime + +class {file_path.split('/')[-1].replace('.py', '').title()}: + ''' + This is a module that handles various operations for the DocsGPT application. + It contains multiple classes and functions for processing data. + ''' + + def __init__(self, config: Dict[str, Any]): + self.config = config + self.initialized = False + self.data_store = {{}} + + def process_data(self, input_data: List[str]) -> Dict[str, Any]: + '''Process input data and return results''' + results = {{}} + for item in input_data: + # Complex processing logic here + processed = self._transform_item(item) + results[item] = processed + return results + + def _transform_item(self, item: str) -> str: + '''Internal transformation logic''' + # Multiple lines of transformation code + transformed = item.upper().strip() + transformed = transformed.replace(' ', '_') + return transformed + + def validate_config(self) -> bool: + '''Validate configuration settings''' + required_keys = ['api_key', 'endpoint', 'model_id'] + return all(key in self.config for key in required_keys) + +# Additional helper functions +def utility_function_one(param: str) -> str: + return param.strip().lower() + +def utility_function_two(data: Dict) -> List: + return list(data.values()) + +def main(): + config = {{'api_key': 'test', 'endpoint': 'http://localhost', 'model_id': 'gpt-4'}} + instance = {file_path.split('/')[-1].replace('.py', '').title()}(config) + instance.process_data(['item1', 'item2', 'item3']) +""" * 2 # Double it to simulate ~1000-1500 tokens per response + + tool_calls.append({ + "call_id": f"call_{i}", + "tool_name": "github_file_scraper", + "action_name": "read_file", + "arguments": {"file_path": file_path}, + "result": {"content": file_content, "status": "success"}, + "status": "success" + }) + + # Add query with all tool calls + queries.append({ + "prompt": "[Agent continues processing]", + "response": "I've scraped 20 Python files. Let me analyze the patterns...", + "tool_calls": tool_calls + }) + + # Add analysis response + queries.append({ + "prompt": "[Agent continues analysis]", + "response": """Based on my analysis of the 20 Python files: + +1. Architecture: The codebase follows a modular architecture with clear separation between API, agents, LLM handlers, and utilities. + +2. Key patterns identified: + - Heavy use of type hints (typing module) + - Consistent error handling patterns + - Service-based architecture for API endpoints + - Factory pattern for LLM creation + - Abstract base classes for extensibility + +3. Core components: + - Agent system with tool integration + - LLM provider abstraction + - Compression service for context management + - Stream processing for real-time responses + +4. Code quality observations: + - Comprehensive docstrings + - Good test coverage + - Clear naming conventions + - Proper separation of concerns""", + "tool_calls": [] + }) + + conversation = {"queries": queries} + + # Mock LLM response for compression + mock_summary = """ + User requested analysis of all Python files in DocsGPT GitHub repository. + Agent scraped 20 files including app.py, API routes, services, agents, and tests. + Analysis revealed modular architecture with service-based design, type hints, + factory patterns, and agent system with tool integration. Code quality is high + with comprehensive docstrings and test coverage. + """ + compression_service.llm.gen.return_value = mock_summary + + # Compress the heavy tool usage + result = compression_service.compress_conversation( + conversation=conversation, + compress_up_to_index=1 # Compress first 2 queries (including all tool calls) + ) + + # Verify compression handled tool calls properly + assert result.query_index == 1 + assert result.compressed_summary is not None + + # Verify the compression prompt included tool call information + call_args = compression_service.llm.gen.call_args + messages = call_args[1]["messages"] + user_message = messages[1]["content"] + + # Should include tool calls section + assert "Tool Calls:" in user_message + assert "github_file_scraper" in user_message + + # Verify compression ratio (should be significant with all that tool data) + original_tokens = result.original_token_count + compressed_tokens = result.compressed_token_count + compression_ratio = result.compression_ratio + + # With 20 large tool responses, original should be substantial + assert original_tokens > 5000 # At least 5k tokens from tool responses + assert compressed_tokens < 500 # Summary should be much smaller + assert compression_ratio > 10 # Should achieve >10x compression + + def test_compression_with_needle_in_haystack(self, compression_service): + """Test compression preserves important information buried in long conversation + + Scenario: User has long conversation with verbose responses, hiding critical + information in the middle. Tests that compression correctly identifies and + preserves key details even when surrounded by lengthy content. + """ + # Create a long conversation with important info buried in the middle + queries = [] + + # Query 1: Long general discussion + queries.append({ + "prompt": "Tell me about Python programming best practices", + "response": """Python best practices encompass a wide range of principles and patterns. +Let me provide a comprehensive overview: + +1. Code Style and Formatting: + - Follow PEP 8 style guide for consistent formatting + - Use 4 spaces for indentation (not tabs) + - Limit lines to 79 characters for code, 72 for docstrings + - Use meaningful variable names that describe their purpose + - Add whitespace around operators and after commas + - Group imports: standard library, third-party, local + +2. Documentation: + - Write clear docstrings for all functions, classes, and modules + - Use type hints for better code clarity and IDE support + - Include examples in docstrings when helpful + - Keep comments up-to-date with code changes + +3. Error Handling: + - Use specific exceptions rather than bare except clauses + - Create custom exceptions for domain-specific errors + - Always clean up resources with context managers (with statement) + - Log errors appropriately for debugging + +4. Testing: + - Write unit tests for all critical functionality + - Aim for high test coverage (80%+) + - Use pytest for modern testing features + - Mock external dependencies in tests + +5. Code Organization: + - Keep functions small and focused on single tasks + - Use classes to group related functionality + - Avoid deep nesting (max 3-4 levels) + - Extract complex conditions into well-named variables + +6. Performance: + - Use list comprehensions for simple transformations + - Avoid premature optimization + - Profile code before optimizing + - Use generators for large datasets + +These practices help maintain readable, maintainable, and efficient code.""", + "tool_calls": [] + }) + + # Query 2: Another long response + queries.append({ + "prompt": "What about Python data structures?", + "response": """Python provides several built-in data structures, each optimized for different use cases: + +1. Lists: + - Ordered, mutable sequences + - Dynamic sizing with amortized O(1) append + - Access by index in O(1) + - Insertion/deletion in middle is O(n) + - Use cases: ordered collections, stacks, queues + - Methods: append(), extend(), insert(), remove(), pop(), sort() + +2. Tuples: + - Ordered, immutable sequences + - Slightly more memory efficient than lists + - Can be used as dictionary keys (if contents are hashable) + - Use cases: fixed collections, function return values, dictionary keys + +3. Dictionaries: + - Unordered (ordered in Python 3.7+) key-value mappings + - Average O(1) lookup, insertion, deletion + - Keys must be hashable + - Use cases: lookups, caching, counting, grouping + - Methods: get(), keys(), values(), items(), update(), pop() + +4. Sets: + - Unordered collections of unique elements + - Average O(1) membership testing + - Efficient for removing duplicates + - Support set operations: union, intersection, difference + - Use cases: membership testing, removing duplicates, set mathematics + +5. Collections module extensions: + - defaultdict: dict with default values for missing keys + - Counter: dict subclass for counting hashable objects + - deque: double-ended queue with O(1) append/pop from both ends + - OrderedDict: maintains insertion order (less relevant in Python 3.7+) + - namedtuple: tuple subclass with named fields + +6. Performance considerations: + - Lists for ordered data with frequent append operations + - Dictionaries for key-based lookups + - Sets for membership testing and uniqueness + - Deques for queue operations from both ends + - Tuples for immutable data + +Understanding these data structures is crucial for writing efficient Python code.""", + "tool_calls": [] + }) + + # Query 3: THE CRITICAL INFORMATION (needle in the haystack) + queries.append({ + "prompt": "I need to remember this important detail", + "response": """I'll make a note of that important detail. + +CRITICAL INFORMATION TO REMEMBER: +The production database password is stored in the environment variable DB_PASSWORD_PROD. +The backup schedule is set to run daily at 3:00 AM UTC. +The API rate limit for premium users is 10,000 requests per hour. +The encryption key rotation happens every 90 days. +The primary contact for incidents is: ops-team@example.com + +I've recorded this information for our conversation. These operational details are important for system administration and should be referenced when needed.""", + "tool_calls": [] + }) + + # Query 4: More long content after the important info + queries.append({ + "prompt": "Explain Python decorators in detail", + "response": """Python decorators are a powerful feature that allows you to modify or enhance functions and classes. Here's a comprehensive explanation: + +1. Basic Concept: + - Decorators are functions that take another function as input + - They return a modified version of that function + - Syntax: @decorator above function definition + - They implement the decorator design pattern + +2. Function Decorators: + ```python + def my_decorator(func): + def wrapper(*args, **kwargs): + # Code before function + result = func(*args, **kwargs) + # Code after function + return result + return wrapper + + @my_decorator + def my_function(): + pass + ``` + +3. Common Use Cases: + - Logging: Record function calls and results + - Timing: Measure execution time + - Authentication: Check permissions before execution + - Caching: Store and return cached results + - Validation: Check input parameters + - Rate limiting: Throttle function calls + +4. Decorators with Arguments: + ```python + def repeat(times): + def decorator(func): + def wrapper(*args, **kwargs): + for _ in range(times): + result = func(*args, **kwargs) + return result + return wrapper + return decorator + + @repeat(3) + def greet(): + print("Hello") + ``` + +5. Class Decorators: + - Can decorate entire classes + - Useful for adding methods or attributes + - Can enforce patterns like singleton + +6. Built-in Decorators: + - @property: Create managed attributes + - @staticmethod: Define static methods + - @classmethod: Define class methods + - @abstractmethod: Define abstract methods + +7. functools.wraps: + - Preserves original function metadata + - Should be used in decorator implementations + - Maintains __name__, __doc__, etc. + +8. Practical Examples: + - @login_required for web routes + - @cache for memoization + - @retry for resilient API calls + - @deprecated for marking old code + +Decorators are essential for writing clean, maintainable Python code with separation of concerns.""", + "tool_calls": [] + }) + + # Query 5: Final long response + queries.append({ + "prompt": "What about Python async programming?", + "response": """Asynchronous programming in Python allows for concurrent execution of I/O-bound operations: + +1. Core Concepts: + - Event loop: Manages and executes async tasks + - Coroutines: Functions defined with async def + - await: Pauses coroutine until awaitable completes + - Tasks: Wrapper for coroutines to run concurrently + +2. Basic Syntax: + ```python + import asyncio + + async def fetch_data(): + await asyncio.sleep(1) + return "data" + + async def main(): + result = await fetch_data() + print(result) + + asyncio.run(main()) + ``` + +3. When to Use Async: + - I/O-bound operations (network requests, file I/O, database queries) + - Multiple concurrent operations + - Real-time applications (websockets, streaming) + - NOT for CPU-bound tasks (use multiprocessing instead) + +4. Common Patterns: + - Gather: Run multiple coroutines concurrently + - create_task: Schedule coroutine execution + - Semaphore: Limit concurrent operations + - Queue: Producer-consumer patterns + +5. Async Libraries: + - aiohttp: Async HTTP client/server + - asyncpg: Async PostgreSQL driver + - motor: Async MongoDB driver + - aioredis: Async Redis client + +6. Error Handling: + - Use try/except in coroutines + - Tasks can be cancelled with task.cancel() + - Timeouts with asyncio.wait_for() + +Understanding async programming is crucial for building scalable Python applications.""", + "tool_calls": [] + }) + + conversation = {"queries": queries} + + # Mock LLM response that MUST preserve the critical information + mock_summary = """ + User asked about Python best practices, data structures, decorators, and async programming. + Discussed code style, testing, documentation standards, and various Python data structures. + + CRITICAL OPERATIONAL DETAILS PROVIDED: + - Production database password stored in DB_PASSWORD_PROD environment variable + - Backup schedule: daily at 3:00 AM UTC + - Premium API rate limit: 10,000 requests/hour + - Encryption key rotation: every 90 days + - Incident contact: ops-team@example.com + + Also covered decorators for code enhancement and async programming for I/O-bound operations. + """ + compression_service.llm.gen.return_value = mock_summary + + # Compress everything except the last query + result = compression_service.compress_conversation( + conversation=conversation, + compress_up_to_index=3 # Compress first 4 queries (includes the critical info) + ) + + # Verify compression happened + assert result.query_index == 3 + assert result.compressed_summary is not None + + # Get the compressed context + conversation["compression_metadata"] = { + "is_compressed": True, + "last_compression_at": datetime.now(timezone.utc), + "compression_points": [result.to_dict()] + } + + summary, recent = compression_service.get_compressed_context( + conversation + ) + + # Verify critical information is in the summary + assert summary is not None + assert "DB_PASSWORD_PROD" in summary or "database password" in summary.lower() + assert "3:00 AM UTC" in summary or "backup" in summary.lower() + assert "10,000" in summary or "rate limit" in summary.lower() + assert "ops-team@example.com" in summary or "incident contact" in summary.lower() + + # Verify only the last query is in recent + assert len(recent) == 1 + assert "async programming" in recent[0]["prompt"].lower() + + # The compression should be substantial (long responses compressed to summary) + assert result.original_token_count > 1300 # 4 long responses + assert result.compressed_token_count < 300 # Summary should be concise + assert result.compression_ratio > 4 # At least 4x compression + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100755 index 00000000..a3588c7e --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,1287 @@ +#!/usr/bin/env python3 +""" +Integration test script for DocsGPT API endpoints. + +Tests: +1. /stream endpoint without agent +2. /api/answer endpoint without agent +3. Create agent via API +4. /stream endpoint with agent +5. /api/answer endpoint with agent + +Usage: + python tests/test_integration.py # auto-generates JWT token from local secret when available + python tests/test_integration.py --base-url http://localhost:7091 + python tests/test_integration.py --token YOUR_JWT_TOKEN # override auto-generation +""" + +import argparse +import json +import os +import sys +import time +from pathlib import Path +from typing import Optional + +import requests + + +class Colors: + """ANSI color codes for terminal output""" + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + + +def generate_default_token() -> tuple[Optional[str], Optional[str]]: + """ + Try to generate a JWT token using the same logic as generate_test_token.py. + Returns a tuple of (token, error_message). Token is None on failure. + """ + secret = os.getenv("JWT_SECRET_KEY") + key_file = Path(".jwt_secret_key") + + if not secret: + try: + secret = key_file.read_text().strip() + except FileNotFoundError: + return None, f"Set JWT_SECRET_KEY or create {key_file} by running the backend once." + except OSError as exc: + return None, f"Could not read {key_file}: {exc}" + + if not secret: + return None, "JWT secret key is empty." + + try: + from jose import jwt # type: ignore + except ImportError: + return None, "python-jose is not installed (pip install 'python-jose' to auto-generate tokens)." + + try: + payload = {"sub": "test_integration_user"} + return jwt.encode(payload, secret, algorithm="HS256"), None + except Exception as exc: + return None, f"Failed to generate JWT token: {exc}" + + +class DocsGPTTester: + def __init__(self, base_url: str, token: Optional[str] = None, token_source: str = "provided"): + self.base_url = base_url.rstrip('/') + self.token = token + self.token_source = token_source + self.headers = {} + if token: + self.headers['Authorization'] = f'Bearer {token}' + self.agent_id = None + self.test_results = [] + + def print_header(self, message: str): + """Print a colored header""" + print(f"\n{Colors.HEADER}{Colors.BOLD}{'=' * 70}{Colors.ENDC}") + print(f"{Colors.HEADER}{Colors.BOLD}{message}{Colors.ENDC}") + print(f"{Colors.HEADER}{Colors.BOLD}{'=' * 70}{Colors.ENDC}\n") + + def print_success(self, message: str): + """Print a success message""" + print(f"{Colors.OKGREEN}✓ {message}{Colors.ENDC}") + + def print_error(self, message: str): + """Print an error message""" + print(f"{Colors.FAIL}✗ {message}{Colors.ENDC}") + + def print_info(self, message: str): + """Print an info message""" + print(f"{Colors.OKCYAN}ℹ {message}{Colors.ENDC}") + + def print_warning(self, message: str): + """Print a warning message""" + print(f"{Colors.WARNING}⚠ {message}{Colors.ENDC}") + + def test_stream_endpoint(self, agent_id: Optional[str] = None) -> bool: + """Test the /stream endpoint""" + endpoint = f"{self.base_url}/stream" + test_name = f"Stream endpoint{'with agent ' + agent_id if agent_id else ' (no agent)'}" + + self.print_header(f"Testing {test_name}") + + payload = { + "question": "What is DocsGPT?", + "history": "[]", + "isNoneDoc": True, + } + + if agent_id: + payload["agent_id"] = agent_id + + try: + self.print_info(f"POST {endpoint}") + self.print_info(f"Payload: {json.dumps(payload, indent=2)}") + + response = requests.post( + endpoint, + json=payload, + headers=self.headers, + stream=True, + timeout=30 + ) + + self.print_info(f"Status Code: {response.status_code}") + + if response.status_code != 200: + self.print_error(f"Expected 200, got {response.status_code}") + self.print_error(f"Response: {response.text[:500]}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + return False + + # Parse SSE stream + events = [] + full_response = "" + conversation_id = None + + for line in response.iter_lines(): + if line: + line = line.decode('utf-8') + if line.startswith('data: '): + data_str = line[6:] # Remove 'data: ' prefix + try: + data = json.loads(data_str) + events.append(data) + + # Handle different event types + if data.get('type') in ['stream', 'answer']: + # Both 'stream' and 'answer' types contain response text + full_response += data.get('message', '') or data.get('answer', '') + elif data.get('type') == 'id': + conversation_id = data.get('id') + elif data.get('type') == 'end': + break + except json.JSONDecodeError: + pass + + self.print_success(f"Received {len(events)} events") + self.print_info(f"Response preview: {full_response[:100]}...") + + if conversation_id: + self.print_success(f"Conversation ID: {conversation_id}") + + if not full_response: + self.print_warning("No response content received") + + self.test_results.append((test_name, True, "Success")) + self.print_success(f"{test_name} passed!") + return True + + except requests.exceptions.RequestException as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + except Exception as e: + self.print_error(f"Unexpected error: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + + def test_answer_endpoint(self, agent_id: Optional[str] = None) -> bool: + """Test the /api/answer endpoint""" + endpoint = f"{self.base_url}/api/answer" + test_name = f"Answer endpoint{' with agent ' + agent_id if agent_id else ' (no agent)'}" + + self.print_header(f"Testing {test_name}") + + payload = { + "question": "What is DocsGPT?", + "history": "[]", + "isNoneDoc": True, + } + + if agent_id: + payload["agent_id"] = agent_id + + try: + self.print_info(f"POST {endpoint}") + self.print_info(f"Payload: {json.dumps(payload, indent=2)}") + + response = requests.post( + endpoint, + json=payload, + headers=self.headers, + timeout=30 + ) + + self.print_info(f"Status Code: {response.status_code}") + + if response.status_code != 200: + self.print_error(f"Expected 200, got {response.status_code}") + self.print_error(f"Response: {response.text[:500]}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + return False + + result = response.json() + + self.print_info(f"Response keys: {list(result.keys())}") + + if 'answer' in result: + answer = result['answer'] + self.print_success(f"Answer received: {answer[:100]}...") + else: + self.print_warning("No 'answer' field in response") + + if 'conversation_id' in result: + self.print_success(f"Conversation ID: {result['conversation_id']}") + + if 'sources' in result: + self.print_info(f"Sources: {len(result['sources'])} items") + + self.test_results.append((test_name, True, "Success")) + self.print_success(f"{test_name} passed!") + return True + + except requests.exceptions.RequestException as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + except Exception as e: + self.print_error(f"Unexpected error: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + + def upload_text_source(self) -> Optional[str]: + """Upload a simple text source for testing + + This creates a source without requiring crawler infrastructure. + """ + endpoint = f"{self.base_url}/api/upload" + test_name = "Upload Text Source" + + self.print_header(f"Testing {test_name}") + + if not self.token: + self.print_warning("No authentication token provided") + self.print_info("Source upload requires authentication") + self.test_results.append((test_name, True, "Skipped (auth required)")) + return None + + # Create a simple text file for upload + test_content = """# DocsGPT Test Documentation + +## Installation + +To install DocsGPT, follow these steps: + +1. Clone the repository +2. Run `docker compose up` +3. Access the application at http://localhost:5173 + +## Configuration + +DocsGPT can be configured using environment variables: +- API_KEY: Your OpenAI API key +- LLM_PROVIDER: Choose between openai, anthropic, or google +- ENABLE_CONVERSATION_COMPRESSION: Enable context compression + +## Features + +DocsGPT provides: +- Conversation compression for long chats +- Real-time token tracking +- Multiple LLM provider support +- Agent system with tools +""" + + try: + self.print_info(f"POST {endpoint}") + self.print_info("Uploading test documentation...") + + # Create a file-like object + files = { + 'file': ('test_docs.txt', test_content.encode(), 'text/plain') + } + data = { + 'user': 'test_user', + 'name': f'Test Docs {int(time.time())}', + } + + response = requests.post( + endpoint, + files=files, + data=data, + headers=self.headers, + timeout=30 + ) + + self.print_info(f"Status Code: {response.status_code}") + + if response.status_code == 200: + result = response.json() + task_id = result.get('task_id') + + if task_id: + self.print_success(f"Upload task started: {task_id}") + self.print_info("Waiting for processing (10 seconds)...") + time.sleep(10) + self.test_results.append((test_name, True, f"Task: {task_id}")) + return task_id + else: + self.print_warning("No task_id returned") + self.test_results.append((test_name, False, "No task_id")) + return None + else: + self.print_error(f"Expected 200, got {response.status_code}") + try: + error_data = response.json() + self.print_error(f"Error: {error_data}") + except Exception: + self.print_error(f"Response: {response.text[:500]}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + return None + + except requests.exceptions.RequestException as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return None + except Exception as e: + self.print_error(f"Unexpected error: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return None + + def upload_crawler_source(self) -> Optional[str]: + """Upload a crawler source for DocsGPT documentation""" + endpoint = f"{self.base_url}/api/remote" + test_name = "Upload Crawler Source" + + self.print_header(f"Testing {test_name}") + + if not self.token: + self.print_warning("No authentication token provided") + self.print_info("Source upload requires authentication") + self.print_info("Skipping source upload and agent tests...") + self.test_results.append((test_name, True, "Skipped (auth required)")) + return None + + payload = { + "user": "test_user", + "source": "crawler", + "name": f"DocsGPT Docs {int(time.time())}", + "data": json.dumps({"url": "https://docs.docsgpt.cloud/"}), + } + + try: + self.print_info(f"POST {endpoint}") + self.print_info("Crawling: https://docs.docsgpt.cloud/") + + response = requests.post( + endpoint, + data=payload, + headers=self.headers, + timeout=30 + ) + + self.print_info(f"Status Code: {response.status_code}") + + if response.status_code == 200: + result = response.json() + task_id = result.get('task_id') + + if task_id: + self.print_success(f"Crawler task started: {task_id}") + self.print_info("Waiting for crawler to complete (30 seconds)...") + time.sleep(30) # Wait for crawler to process + self.test_results.append((test_name, True, f"Task: {task_id}")) + return task_id + else: + self.print_warning("No task_id returned") + self.test_results.append((test_name, False, "No task_id")) + return None + else: + self.print_error(f"Expected 200, got {response.status_code}") + try: + error_data = response.json() + self.print_error(f"Error: {error_data}") + except Exception: + self.print_error(f"Response: {response.text[:500]}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + return None + + except requests.exceptions.RequestException as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return None + except Exception as e: + self.print_error(f"Unexpected error: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return None + + def get_source_id_from_task(self, task_id: str) -> Optional[str]: + """Check task status and get source ID""" + endpoint = f"{self.base_url}/api/task_status" + + try: + response = requests.get( + endpoint, + params={"task_id": task_id}, + headers=self.headers, + timeout=10 + ) + + if response.status_code == 200: + result = response.json() + if result.get('status') == 'SUCCESS': + # Task completed, now find the source + # Query sources collection to find the latest source + sources_response = requests.get( + f"{self.base_url}/api/sources", + headers=self.headers, + timeout=10 + ) + if sources_response.status_code == 200: + sources = sources_response.json() + # Filter out the "Default" source and get user sources only + user_sources = [s for s in sources if s.get('date') != 'default'] + if user_sources and len(user_sources) > 0: + # Get the most recent source (first one, as they're sorted by date desc) + latest_source = user_sources[0] + return latest_source.get('id') + return None + except Exception as e: + self.print_error(f"Error getting source ID: {str(e)}") + return None + + def create_agent(self, source_id: Optional[str] = None, published: bool = False) -> Optional[tuple]: + """Create an agent via API + + Args: + source_id: Optional source ID to attach to agent + published: If True, create published agent (requires source_id) + + Returns: + Tuple of (agent_id, api_key) if successful, None otherwise + """ + endpoint = f"{self.base_url}/api/create_agent" + + if published and source_id: + test_name = f"Create Published Agent with source {source_id[:8]}..." + elif published: + test_name = "Create Published Agent (skipped - no source)" + else: + test_name = "Create Draft Agent" + + self.print_header(f"Testing {test_name}") + + if not self.token: + self.print_warning("No authentication token provided") + self.print_info("Agent creation requires authentication") + self.print_info("To test agents, provide a JWT token with --token argument") + self.print_info("Skipping agent tests...") + # Mark as skipped rather than attempting without auth + self.test_results.append((test_name, True, "Skipped (auth required)")) + return None + + # Published agents require a source + if published and not source_id: + self.print_warning("Cannot create published agent without source") + self.test_results.append((test_name, True, "Skipped (no source)")) + return None + + # Create payload based on type + if published: + self.print_info(f"Creating published agent with source {source_id[:8]}...") + payload = { + "name": f"Test Agent (Published) {int(time.time())}", + "description": "Integration test agent with source", + "prompt_id": "default", + "chunks": 2, + "retriever": "classic", + "agent_type": "classic", + "status": "published", + "source": source_id, + } + else: + self.print_info("Creating draft agent (for agent_id testing)") + payload = { + "name": f"Test Agent (Draft) {int(time.time())}", + "description": "Integration test draft agent", + "prompt_id": "default", + "chunks": 2, + "retriever": "classic", + "agent_type": "classic", + "status": "draft", + } + + try: + self.print_info(f"POST {endpoint}") + self.print_info(f"Payload: {json.dumps(payload, indent=2)}") + + response = requests.post( + endpoint, + json=payload, + headers=self.headers, + timeout=10 + ) + + self.print_info(f"Status Code: {response.status_code}") + + if response.status_code in [200, 201]: # Accept both 200 OK and 201 Created + result = response.json() + agent_id = result.get('id') + api_key = result.get('key', '') + + if agent_id: + self.agent_id = agent_id + self.print_success(f"Agent created with ID: {agent_id}") + if api_key: + self.print_success(f"Agent API key: {api_key[:20]}...") + self.test_results.append((test_name, True, f"ID: {agent_id}, API Key: Yes")) + return (agent_id, api_key) + else: + self.print_warning("Agent created but no API key (draft agent)") + self.test_results.append((test_name, True, f"ID: {agent_id}, API Key: No")) + return (agent_id, None) + else: + self.print_warning("Agent created but no ID returned") + self.test_results.append((test_name, False, "No ID returned")) + return None + elif response.status_code == 401: + self.print_warning("Authentication required for agent creation") + self.print_info("To test agents, provide a JWT token with --token argument") + self.print_info("Skipping agent tests...") + # Mark as "skipped" rather than "failed" + self.test_results.append((test_name, True, "Skipped (auth required)")) + return None + else: + self.print_error(f"Expected 200/201, got {response.status_code}") + try: + error_data = response.json() + self.print_error(f"Error: {error_data.get('message', response.text[:200])}") + except Exception: + self.print_error(f"Response: {response.text[:500]}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + return None + + except requests.exceptions.RequestException as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return None + except Exception as e: + self.print_error(f"Unexpected error: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return None + + def test_api_key_endpoint(self, api_key: str, endpoint_type: str = "stream") -> bool: + """Test endpoint with API key instead of agent_id""" + test_name = f"{endpoint_type.capitalize()} endpoint with API key" + + self.print_header(f"Testing {test_name}") + + if endpoint_type == "stream": + endpoint = f"{self.base_url}/stream" + else: + endpoint = f"{self.base_url}/api/answer" + + payload = { + "question": "What is DocsGPT?", + "history": "[]", + "api_key": api_key, # Use api_key instead of agent_id + } + + try: + self.print_info(f"POST {endpoint}") + self.print_info(f"Using API key: {api_key[:20]}...") + + if endpoint_type == "stream": + response = requests.post( + endpoint, + json=payload, + headers=self.headers, + stream=True, + timeout=30 + ) + + self.print_info(f"Status Code: {response.status_code}") + + if response.status_code != 200: + self.print_error(f"Expected 200, got {response.status_code}") + self.print_error(f"Response: {response.text[:500]}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + return False + + # Parse SSE stream + events = [] + full_response = "" + + for line in response.iter_lines(): + if line: + line = line.decode('utf-8') + if line.startswith('data: '): + data_str = line[6:] + try: + data = json.loads(data_str) + events.append(data) + + if data.get('type') in ['stream', 'answer']: + full_response += data.get('message', '') or data.get('answer', '') + elif data.get('type') == 'end': + break + except json.JSONDecodeError: + pass + + self.print_success(f"Received {len(events)} events") + self.print_info(f"Response preview: {full_response[:100]}...") + self.test_results.append((test_name, True, "Success")) + return True + + else: # answer endpoint + response = requests.post( + endpoint, + json=payload, + headers=self.headers, + timeout=30 + ) + + self.print_info(f"Status Code: {response.status_code}") + + if response.status_code != 200: + self.print_error(f"Expected 200, got {response.status_code}") + self.print_error(f"Response: {response.text[:500]}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + return False + + result = response.json() + answer = result.get('answer', '') + self.print_success(f"Answer received: {answer[:100]}...") + self.test_results.append((test_name, True, "Success")) + return True + + except requests.exceptions.RequestException as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + except Exception as e: + self.print_error(f"Unexpected error: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + + def test_model_validation(self) -> bool: + """Test model_id validation""" + endpoint = f"{self.base_url}/stream" + test_name = "Model validation (invalid model_id)" + + self.print_header(f"Testing {test_name}") + + payload = { + "question": "Test question", + "history": "[]", + "isNoneDoc": True, + "model_id": "invalid-model-xyz-123", + } + + try: + self.print_info(f"POST {endpoint}") + self.print_info("Testing with invalid model_id: invalid-model-xyz-123") + + response = requests.post( + endpoint, + json=payload, + headers=self.headers, + stream=True, + timeout=10 + ) + + self.print_info(f"Status Code: {response.status_code}") + + if response.status_code == 400: + # Read the error from SSE stream + error_message = None + error_field = None + for line in response.iter_lines(): + if line: + line = line.decode('utf-8') + if line.startswith('data: '): + data_str = line[6:] + try: + data = json.loads(data_str) + if data.get('type') == 'error': + # Try both 'message' and 'error' fields + error_message = data.get('message') or data.get('error', '') + error_field = 'message' if 'message' in data else 'error' + break + except json.JSONDecodeError: + pass + + # Consider it successful if we got a 400 with any error message + if error_message: + self.print_success("Invalid model_id rejected with 400 status") + self.print_info(f"Error ({error_field}): {error_message[:200]}") + + # Check if it's the detailed validation error or generic error + if 'Invalid model_id' in error_message or 'model' in error_message.lower(): + self.print_success("✓ Validation error contains model information") + self.test_results.append((test_name, True, "Validation works")) + else: + self.print_warning("Generic error message (validation may need improvement)") + self.test_results.append((test_name, True, "Generic validation")) + return True + else: + self.print_warning("No error message in response") + self.test_results.append((test_name, False, "No error message")) + return False + else: + self.print_warning(f"Expected 400, got {response.status_code}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + return False + + except Exception as e: + self.print_error(f"Unexpected error: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + + def create_web_scraping_agent(self) -> Optional[tuple]: + """Create an agent with read_webpage tool enabled + + Returns: + Tuple of (agent_id, api_key) if successful, None otherwise + """ + endpoint = f"{self.base_url}/api/create_agent" + test_name = "Create Web Scraping Agent" + + self.print_header(f"Testing {test_name}") + + if not self.token: + self.print_warning("No authentication token provided") + self.test_results.append((test_name, True, "Skipped (auth required)")) + return None + + # Create agent with read_webpage tool + payload = { + "name": f"Web Scraping Agent {int(time.time())}", + "description": "Test agent with read_webpage tool for compression testing", + "prompt_id": "default", + "chunks": 2, + "retriever": "classic", + "agent_type": "react", # ReAct agent supports tools + "status": "draft", + "tools": ["read_webpage"], # Enable read_webpage tool + } + + try: + self.print_info(f"POST {endpoint}") + self.print_info("Creating agent with read_webpage tool...") + + response = requests.post( + endpoint, + json=payload, + headers=self.headers, + timeout=10 + ) + + self.print_info(f"Status Code: {response.status_code}") + + if response.status_code in [200, 201]: + result = response.json() + agent_id = result.get('id') + api_key = result.get('key', '') + + if agent_id: + self.print_success(f"Web scraping agent created with ID: {agent_id}") + if api_key: + self.print_success(f"Agent API key: {api_key[:20]}...") + self.test_results.append((test_name, True, f"ID: {agent_id}, API Key: Yes")) + return (agent_id, api_key) + else: + self.print_warning("Agent created but no API key (draft agent)") + self.test_results.append((test_name, True, f"ID: {agent_id}, API Key: No")) + return (agent_id, None) + else: + self.print_warning("Agent created but no ID returned") + self.test_results.append((test_name, False, "No ID returned")) + return None + else: + self.print_error(f"Expected 200/201, got {response.status_code}") + try: + error_data = response.json() + self.print_error(f"Error: {error_data.get('message', response.text[:200])}") + except Exception: + self.print_error(f"Response: {response.text[:500]}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + return None + + except requests.exceptions.RequestException as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return None + except Exception as e: + self.print_error(f"Unexpected error: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return None + + def test_compression_heavy_tool_usage(self, agent_result: Optional[tuple] = None) -> bool: + """Test compression with heavy tool usage (real API calls) + + This simulates a scenario where an agent makes many tool calls + (including read_webpage for web scraping), generating large responses + that should trigger compression. + + Args: + agent_result: Optional tuple of (agent_id, api_key) from agent creation + """ + endpoint = f"{self.base_url}/api/answer" + test_name = "Compression - Heavy Tool Usage" + + self.print_header(f"Testing {test_name}") + + if not self.token: + self.print_warning("Authentication required for compression tests") + self.test_results.append((test_name, True, "Skipped (auth required)")) + return False + + # Use provided agent or create one + if not agent_result: + self.print_info("No web scraping agent provided, creating one...") + agent_result = self.create_web_scraping_agent() + + if not agent_result: + self.print_warning("Could not create web scraping agent, using isNoneDoc instead") + agent_id = None + api_key = None + else: + agent_id, api_key = agent_result + + # Define URLs to scrape for testing + urls_to_scrape = [ + "https://docs.docsgpt.cloud/", + "https://docs.docsgpt.cloud/getting-started/quickstart", + "https://docs.docsgpt.cloud/getting-started/installation", + "https://docs.docsgpt.cloud/extensions/extensions-intro", + "https://github.com/arc53/DocsGPT", + ] + + # Make requests with tool usage + self.print_info("Making 10 consecutive requests to build up conversation history...") + self.print_info("Some requests will use read_webpage tool for web scraping...") + + current_conv_id = None + + for i in range(10): + # Alternate between regular questions and web scraping + if i < 5 and agent_id: + # Use web scraping for first 5 requests + url = urls_to_scrape[i % len(urls_to_scrape)] + question = f"Please read and summarize the content from this webpage: {url}" + else: + # Use regular questions for remaining requests + question = f"Tell me about Python topic number {i+1}: data structures, decorators, async, testing, etc. Please provide a comprehensive explanation." + + payload = { + "question": question, + "history": "[]", + "model_id": "gemini-2.5-pro", + } + + # Use agent if available, otherwise isNoneDoc + if agent_id: + payload["agent_id"] = agent_id + elif api_key: + payload["api_key"] = api_key + else: + payload["isNoneDoc"] = True + + if current_conv_id: + payload["conversation_id"] = current_conv_id + + try: + response = requests.post( + endpoint, + json=payload, + headers=self.headers, + timeout=90 # Longer timeout for web scraping + ) + + if response.status_code == 200: + result = response.json() + current_conv_id = result.get('conversation_id', current_conv_id) + answer_preview = result.get('answer', '')[:80] + self.print_success(f"Request {i+1}/10 completed (conv_id: {current_conv_id})") + self.print_info(f" Answer preview: {answer_preview}...") + else: + self.print_error(f"Request {i+1}/10 failed with status {response.status_code}") + self.test_results.append((test_name, False, f"Request {i+1} failed")) + return False + + time.sleep(2) # Small delay between requests + + except Exception as e: + self.print_error(f"Request {i+1}/10 failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + + # Check if conversation was compressed by examining metadata + if current_conv_id: + self.print_info(f"Checking compression status for conversation {current_conv_id}") + # Note: This would require a /api/conversation/{id} endpoint to verify + self.print_success("Heavy tool usage test completed") + tool_info = "with read_webpage" if agent_id else "without tools" + self.test_results.append((test_name, True, f"10 requests {tool_info}, conv_id: {current_conv_id}")) + return True + else: + self.print_warning("No conversation_id received") + self.test_results.append((test_name, False, "No conversation_id")) + return False + + def test_compression_needle_in_haystack(self) -> bool: + """Test that compression preserves critical information + + This sends a long conversation with important info in the middle, + then asks about that info to verify it was preserved through compression. + """ + endpoint = f"{self.base_url}/api/answer" + test_name = "Compression - Needle in Haystack" + + self.print_header(f"Testing {test_name}") + + if not self.token: + self.print_warning("Authentication required for compression tests") + self.test_results.append((test_name, True, "Skipped (auth required)")) + return False + + conversation_id = None + + # Step 1: Send general questions + self.print_info("Step 1: Sending general questions...") + for i, question in enumerate([ + "Tell me about Python best practices in detail", + "Explain Python data structures comprehensively", + ]): + payload = { + "question": question, + "history": "[]", + "isNoneDoc": True, + "model_id": "gemini-2.5-pro", + } + + if conversation_id: + payload["conversation_id"] = conversation_id + + try: + response = requests.post(endpoint, json=payload, headers=self.headers, timeout=60) + if response.status_code == 200: + result = response.json() + conversation_id = result.get('conversation_id', conversation_id) + self.print_success(f"General question {i+1}/2 completed") + else: + self.print_error(f"Request failed with status {response.status_code}") + self.test_results.append((test_name, False, "General questions failed")) + return False + time.sleep(2) + except Exception as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + + # Step 2: Send CRITICAL information + self.print_info("Step 2: Sending CRITICAL information to remember...") + critical_payload = { + "question": "Please remember this critical information: The production database password is stored in DB_PASSWORD_PROD environment variable. The backup runs at 3:00 AM UTC daily. Premium users have 10,000 req/hour limit.", + "history": "[]", + "isNoneDoc": True, + "model_id": "gemini-2.5-pro", + "conversation_id": conversation_id, + } + + try: + response = requests.post(endpoint, json=critical_payload, headers=self.headers, timeout=60) + if response.status_code == 200: + result = response.json() + conversation_id = result.get('conversation_id', conversation_id) + self.print_success("Critical information sent") + else: + self.print_error("Critical info request failed") + self.test_results.append((test_name, False, "Critical info failed")) + return False + time.sleep(2) + except Exception as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + + # Step 3: Send more general questions to bury the critical info + self.print_info("Step 3: Sending more questions to bury the critical info...") + for i, question in enumerate([ + "Explain Python decorators in great detail", + "Tell me about Python async programming comprehensively", + ]): + payload = { + "question": question, + "history": "[]", + "isNoneDoc": True, + "model_id": "gemini-2.5-pro", + "conversation_id": conversation_id, + } + + try: + response = requests.post(endpoint, json=payload, headers=self.headers, timeout=60) + if response.status_code == 200: + result = response.json() + conversation_id = result.get('conversation_id', conversation_id) + self.print_success(f"Burying question {i+1}/2 completed") + else: + self.print_error("Request failed") + self.test_results.append((test_name, False, "Burying questions failed")) + return False + time.sleep(2) + except Exception as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + + # Step 4: Ask about the critical information + self.print_info("Step 4: Testing if critical info was preserved...") + recall_payload = { + "question": "What was the database password environment variable I mentioned earlier?", + "history": "[]", + "isNoneDoc": True, + "model_id": "gemini-2.5-pro", + "conversation_id": conversation_id, + } + + try: + response = requests.post(endpoint, json=recall_payload, headers=self.headers, timeout=60) + if response.status_code == 200: + result = response.json() + answer = result.get('answer', '').lower() + + # Check if the critical info was preserved + if 'db_password_prod' in answer or 'database password' in answer: + self.print_success("✓ Critical information preserved through compression!") + self.print_info(f"Answer: {answer[:150]}...") + self.test_results.append((test_name, True, "Info preserved")) + return True + else: + self.print_warning("Critical information may have been lost") + self.print_info(f"Answer: {answer[:150]}...") + self.test_results.append((test_name, False, "Info not preserved")) + return False + else: + self.print_error("Recall request failed") + self.test_results.append((test_name, False, "Recall failed")) + return False + except Exception as e: + self.print_error(f"Request failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + return False + + def print_summary(self): + """Print test results summary""" + self.print_header("Test Results Summary") + + passed = sum(1 for _, success, _ in self.test_results if success) + failed = len(self.test_results) - passed + + print(f"\n{Colors.BOLD}Total Tests: {len(self.test_results)}{Colors.ENDC}") + print(f"{Colors.OKGREEN}Passed: {passed}{Colors.ENDC}") + print(f"{Colors.FAIL}Failed: {failed}{Colors.ENDC}\n") + + print(f"{Colors.BOLD}Detailed Results:{Colors.ENDC}") + for test_name, success, message in self.test_results: + status = f"{Colors.OKGREEN}PASS{Colors.ENDC}" if success else f"{Colors.FAIL}FAIL{Colors.ENDC}" + print(f" {status} - {test_name}: {message}") + + print() + return failed == 0 + + def run_all_tests(self): + """Run all integration tests""" + self.print_header("DocsGPT Integration Tests") + self.print_info(f"Base URL: {self.base_url}") + if self.token: + self.print_info(f"Authentication: Yes ({self.token_source})") + else: + self.print_info("Authentication: No (agent-related tests will be skipped)") + + # Test 1: Stream endpoint without agent + self.test_stream_endpoint() + time.sleep(1) + + # Test 2: Answer endpoint without agent + self.test_answer_endpoint() + time.sleep(1) + + # Test 3: Model validation + self.test_model_validation() + time.sleep(1) + + # Test 4: Compression tests (requires token) + if self.token: + self.print_info("Running compression integration tests...") + time.sleep(1) + + # Test 4a: Heavy tool usage compression + self.test_compression_heavy_tool_usage() + time.sleep(2) + + # Test 4b: Needle in haystack compression + self.test_compression_needle_in_haystack() + time.sleep(1) + else: + self.print_info("Skipping compression tests (no authentication)") + + # Test 5: Upload text source (requires token) - faster than crawler + task_id = self.upload_text_source() + source_id = None + + if task_id: + # Test 6: Get source ID from completed task + source_id = self.get_source_id_from_task(task_id) + if source_id: + self.print_success(f"Source created with ID: {source_id}") + else: + self.print_warning("Could not retrieve source ID from task - trying crawler fallback") + # Fallback to crawler if text upload failed + crawler_task_id = self.upload_crawler_source() + if crawler_task_id: + source_id = self.get_source_id_from_task(crawler_task_id) + if source_id: + self.print_success(f"Source created with ID (crawler): {source_id}") + else: + self.print_warning("Could not retrieve source ID from crawler task either") + + # Test 7: Create published agent (for API key testing) - default behavior + # Published agents get an API key automatically + published_result = self.create_agent(source_id=source_id, published=True) + + if published_result: + agent_id, api_key = published_result + time.sleep(1) + + if api_key: + # Test 8 & 9: Test with API key (primary method) + self.test_api_key_endpoint(api_key, endpoint_type="stream") + time.sleep(1) + self.test_api_key_endpoint(api_key, endpoint_type="answer") + time.sleep(1) + + # Test 10: Also test with agent_id for completeness + self.test_stream_endpoint(agent_id=agent_id) + time.sleep(1) + self.test_answer_endpoint(agent_id=agent_id) + + # Test 11: If agent has a source, test source-specific questions + if source_id: + time.sleep(1) + self.print_info("Testing published agent with source-specific questions...") + + test_name = "Published agent with source (DocsGPT question)" + self.print_header(f"Testing {test_name}") + + payload = { + "question": "How do I install DocsGPT?", + "history": "[]", + "api_key": api_key, + } + + try: + response = requests.post( + f"{self.base_url}/api/answer", + json=payload, + headers=self.headers, + timeout=30 + ) + + if response.status_code == 200: + result = response.json() + answer = result.get('answer', '') + self.print_success(f"Answer received: {answer[:100]}...") + + if any(word in answer.lower() for word in ['install', 'docker', 'setup']): + self.print_success("Answer contains relevant information from source") + self.test_results.append((test_name, True, "Success")) + else: + self.print_warning("Answer may not be using source data") + self.test_results.append((test_name, True, "Answer unclear")) + else: + self.print_error(f"Status {response.status_code}") + self.test_results.append((test_name, False, f"Status {response.status_code}")) + + except Exception as e: + self.print_error(f"Test failed: {str(e)}") + self.test_results.append((test_name, False, str(e))) + else: + self.print_warning("Published agent created but no API key received") + self.print_info("Testing with agent_id instead...") + # Fallback to agent_id testing + self.test_stream_endpoint(agent_id=agent_id) + time.sleep(1) + self.test_answer_endpoint(agent_id=agent_id) + else: + if self.token: + self.print_warning("Published agent creation failed - some tests skipped") + else: + self.print_info("Skipping agent tests (no authentication token)") + + # Print summary + success = self.print_summary() + return 0 if success else 1 + + +def main(): + parser = argparse.ArgumentParser( + description='Integration test script for DocsGPT API endpoints', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Test local instance + python tests/test_integration.py # auto-generates JWT token from local secret if possible + + # Test remote instance + python tests/test_integration.py --base-url https://app.docsgpt.com + + # Test with authentication (required for agent creation) + python tests/test_integration.py --token YOUR_JWT_TOKEN + + # Test specific endpoint only + python tests/test_integration.py --base-url http://localhost:7091 --token YOUR_TOKEN + """ + ) + + parser.add_argument( + '--base-url', + default='http://localhost:7091', + help='Base URL of DocsGPT instance (default: http://localhost:7091)' + ) + + parser.add_argument( + '--token', + help='JWT authentication token (auto-generated from local secret when available)' + ) + + args = parser.parse_args() + + token = args.token + token_source = "provided via --token" if token else "auto-generated from local JWT secret" + + if not token: + token, token_error = generate_default_token() + if token: + print(f"{Colors.OKCYAN}ℹ Using auto-generated JWT token from local secret{Colors.ENDC}") + else: + token_source = "none" + if token_error: + print(f"{Colors.WARNING}⚠ Could not auto-generate JWT token: {token_error}{Colors.ENDC}") + print(f"{Colors.WARNING}⚠ Agent creation tests will be skipped unless you provide --token{Colors.ENDC}") + + try: + tester = DocsGPTTester(args.base_url, token, token_source=token_source) + exit_code = tester.run_all_tests() + sys.exit(exit_code) + except KeyboardInterrupt: + print(f"\n{Colors.WARNING}Tests interrupted by user{Colors.ENDC}") + sys.exit(1) + except Exception as e: + print(f"\n{Colors.FAIL}Fatal error: {str(e)}{Colors.ENDC}") + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/tests/test_model_validation.py b/tests/test_model_validation.py new file mode 100644 index 00000000..379ccbf6 --- /dev/null +++ b/tests/test_model_validation.py @@ -0,0 +1,106 @@ +""" +Tests for model validation and base_url functionality +""" +import pytest +from application.core.model_settings import ( + AvailableModel, + ModelCapabilities, + ModelProvider, + ModelRegistry, +) +from application.core.model_utils import ( + get_base_url_for_model, + validate_model_id, +) + + +@pytest.mark.unit +def test_model_with_base_url(): + """Test that AvailableModel can store and retrieve base_url""" + model = AvailableModel( + id="test-model", + provider=ModelProvider.OPENAI, + display_name="Test Model", + description="Test model with custom base URL", + base_url="https://custom-endpoint.com/v1", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=8192, + ), + ) + + assert model.base_url == "https://custom-endpoint.com/v1" + assert model.id == "test-model" + assert model.provider == ModelProvider.OPENAI + + # Test to_dict includes base_url + model_dict = model.to_dict() + assert "base_url" in model_dict + assert model_dict["base_url"] == "https://custom-endpoint.com/v1" + + +@pytest.mark.unit +def test_model_without_base_url(): + """Test that models without base_url still work""" + model = AvailableModel( + id="test-model-no-url", + provider=ModelProvider.OPENAI, + display_name="Test Model", + description="Test model without base URL", + capabilities=ModelCapabilities( + supports_tools=True, + context_window=8192, + ), + ) + + assert model.base_url is None + + # Test to_dict doesn't include base_url when None + model_dict = model.to_dict() + assert "base_url" not in model_dict + + +@pytest.mark.unit +def test_validate_model_id(): + """Test model_id validation""" + # Get the registry instance to check what models are available + ModelRegistry.get_instance() + + # Test with a model that should exist (docsgpt-local is always added) + assert validate_model_id("docsgpt-local") is True + + # Test with invalid model_id + assert validate_model_id("invalid-model-xyz-123") is False + + # Test with None + assert validate_model_id(None) is False + + +@pytest.mark.unit +def test_get_base_url_for_model(): + """Test retrieving base_url for a model""" + # Test with a model that doesn't have base_url + result = get_base_url_for_model("docsgpt-local") + assert result is None # docsgpt-local doesn't have custom base_url + + # Test with invalid model + result = get_base_url_for_model("invalid-model") + assert result is None + + +@pytest.mark.unit +def test_model_validation_error_message(): + """Test that validation provides helpful error messages""" + from application.api.answer.services.stream_processor import StreamProcessor + + # Create processor with invalid model_id + data = {"model_id": "invalid-model-xyz"} + processor = StreamProcessor(data, None) + + # Should raise ValueError with helpful message + with pytest.raises(ValueError) as exc_info: + processor._validate_and_set_model() + + error_msg = str(exc_info.value) + assert "Invalid model_id 'invalid-model-xyz'" in error_msg + assert "Available models:" in error_msg diff --git a/tests/test_token_management.py b/tests/test_token_management.py new file mode 100644 index 00000000..0b166953 --- /dev/null +++ b/tests/test_token_management.py @@ -0,0 +1,314 @@ +""" +Tests for token management and compression features. + +NOTE: These tests are for future planned features that are not yet implemented. +They are skipped until the following modules are created: +- application.compression (DocumentCompressor, HistoryCompressor, etc.) +- application.core.token_budget (TokenBudgetManager) +""" +# ruff: noqa: F821 +import pytest + +pytest.skip( + "Token management features not yet implemented - planned for future release", + allow_module_level=True, +) + + +class TestTokenBudgetManager: + """Test TokenBudgetManager functionality""" + + def test_calculate_budget(self): + """Test budget calculation""" + manager = TokenBudgetManager(model_id="gpt-4o") + budget = manager.calculate_budget() + + assert budget.total_budget > 0 + assert budget.system_prompt > 0 + assert budget.chat_history > 0 + assert budget.retrieved_docs > 0 + + def test_measure_usage(self): + """Test token usage measurement""" + manager = TokenBudgetManager(model_id="gpt-4o") + + usage = manager.measure_usage( + system_prompt="You are a helpful assistant.", + current_query="What is Python?", + chat_history=[ + {"prompt": "Hello", "response": "Hi there!"}, + {"prompt": "How are you?", "response": "I'm doing well, thanks!"}, + ], + ) + + assert usage.total > 0 + assert usage.system_prompt > 0 + assert usage.current_query > 0 + assert usage.chat_history > 0 + + def test_compression_recommendation(self): + """Test compression recommendation generation""" + manager = TokenBudgetManager(model_id="gpt-4o") + + # Create scenario with excessive history + large_history = [ + {"prompt": f"Question {i}" * 100, "response": f"Answer {i}" * 100} + for i in range(100) + ] + + budget, usage, recommendation = manager.check_and_recommend( + system_prompt="You are a helpful assistant.", + current_query="What is Python?", + chat_history=large_history, + ) + + # Should recommend compression + assert recommendation.needs_compression() + assert recommendation.compress_history + + +class TestHistoryCompressor: + """Test HistoryCompressor functionality""" + + def test_sliding_window_compression(self): + """Test sliding window compression strategy""" + compressor = HistoryCompressor() + + history = [ + {"prompt": f"Question {i}", "response": f"Answer {i}"} for i in range(20) + ] + + compressed, metadata = compressor.compress( + history, target_tokens=500, strategy="sliding_window" + ) + + assert len(compressed) < len(history) + assert metadata["original_messages"] == 20 + assert metadata["compressed_messages"] < 20 + assert metadata["strategy"] == "sliding_window" + + def test_preserve_tool_calls(self): + """Test that tool calls are preserved during compression""" + compressor = HistoryCompressor() + + history = [ + {"prompt": "Question 1", "response": "Answer 1"}, + { + "prompt": "Use a tool", + "response": "Tool used", + "tool_calls": [{"tool_name": "search", "result": "Found something"}], + }, + {"prompt": "Question 3", "response": "Answer 3"}, + ] + + compressed, metadata = compressor.compress( + history, target_tokens=200, strategy="sliding_window", preserve_tool_calls=True + ) + + # Tool call message should be preserved + has_tool_calls = any("tool_calls" in msg for msg in compressed) + assert has_tool_calls + + +class TestDocumentCompressor: + """Test DocumentCompressor functionality""" + + def test_rerank_compression(self): + """Test re-ranking compression strategy""" + compressor = DocumentCompressor() + + docs = [ + {"text": f"Document {i} with some content here" * 20, "title": f"Doc {i}"} + for i in range(10) + ] + + compressed, metadata = compressor.compress( + docs, target_tokens=500, query="Document 5", strategy="rerank" + ) + + assert len(compressed) < len(docs) + assert metadata["original_docs"] == 10 + assert metadata["strategy"] == "rerank" + + def test_excerpt_extraction(self): + """Test excerpt extraction strategy""" + compressor = DocumentCompressor() + + docs = [ + { + "text": "This is a long document. " * 100 + + "Python is great. " + + "More text here. " * 100, + "title": "Python Guide", + } + ] + + compressed, metadata = compressor.compress( + docs, target_tokens=300, query="Python", strategy="excerpt" + ) + + assert metadata["excerpts_created"] > 0 + # Excerpt should contain the query term + assert "python" in compressed[0]["text"].lower() + + +class TestToolResultCompressor: + """Test ToolResultCompressor functionality""" + + def test_truncate_large_results(self): + """Test truncation of large tool results""" + compressor = ToolResultCompressor() + + tool_results = [ + { + "tool_name": "search", + "result": "Very long result " * 1000, + "arguments": {}, + } + ] + + compressed, metadata = compressor.compress( + tool_results, target_tokens=100, strategy="truncate" + ) + + assert metadata["results_truncated"] > 0 + # Result should be shorter + compressed_result_len = len(str(compressed[0]["result"])) + original_result_len = len(tool_results[0]["result"]) + assert compressed_result_len < original_result_len + + def test_extract_json_fields(self): + """Test extraction of key fields from JSON results""" + compressor = ToolResultCompressor() + + tool_results = [ + { + "tool_name": "api_call", + "result": { + "data": {"important": "value"}, + "metadata": {"verbose": "information" * 100}, + "debug": {"lots": "of data" * 100}, + }, + "arguments": {}, + } + ] + + compressed, metadata = compressor.compress( + tool_results, target_tokens=100, strategy="extract" + ) + + # Should keep important fields, discard verbose ones + assert "data" in compressed[0]["result"] + + +class TestPromptOptimizer: + """Test PromptOptimizer functionality""" + + def test_compress_tool_descriptions(self): + """Test compression of tool descriptions""" + optimizer = PromptOptimizer() + + tools = [ + { + "type": "function", + "function": { + "name": f"tool_{i}", + "description": "This is a very long description " * 50, + "parameters": {}, + }, + } + for i in range(10) + ] + + optimized, metadata = optimizer.optimize_tools( + tools, target_tokens=500, strategy="compress" + ) + + assert metadata["optimized_tokens"] < metadata["original_tokens"] + assert metadata["descriptions_compressed"] > 0 + + def test_lazy_load_tools(self): + """Test lazy loading of tools based on query""" + optimizer = PromptOptimizer() + + tools = [ + { + "type": "function", + "function": { + "name": "search_tool", + "description": "Search for information", + "parameters": {}, + }, + }, + { + "type": "function", + "function": { + "name": "calculate_tool", + "description": "Perform calculations", + "parameters": {}, + }, + }, + { + "type": "function", + "function": { + "name": "other_tool", + "description": "Do something else", + "parameters": {}, + }, + }, + ] + + optimized, metadata = optimizer.optimize_tools( + tools, target_tokens=200, query="I want to search for something", strategy="lazy_load" + ) + + # Should prefer search tool + assert len(optimized) < len(tools) + tool_names = [t["function"]["name"] for t in optimized] + # Search tool should be included due to query relevance + assert any("search" in name for name in tool_names) + + +def test_integration_compression_workflow(): + """Test complete compression workflow""" + # Simulate a scenario with large inputs + manager = TokenBudgetManager(model_id="gpt-4o") + history_compressor = HistoryCompressor() + doc_compressor = DocumentCompressor() + + # Large chat history + history = [ + {"prompt": f"Question {i}" * 50, "response": f"Answer {i}" * 50} + for i in range(50) + ] + + # Large documents + docs = [ + {"text": f"Document {i} content" * 100, "title": f"Doc {i}"} for i in range(20) + ] + + # Check budget + budget, usage, recommendation = manager.check_and_recommend( + system_prompt="You are a helpful assistant.", + current_query="What is Python?", + chat_history=history, + retrieved_docs=docs, + ) + + # Should need compression + assert recommendation.needs_compression() + + # Apply compression + if recommendation.compress_history: + compressed_history, hist_meta = history_compressor.compress( + history, recommendation.target_history_tokens or budget.chat_history + ) + assert len(compressed_history) < len(history) + + if recommendation.compress_docs: + compressed_docs, doc_meta = doc_compressor.compress( + docs, + recommendation.target_docs_tokens or budget.retrieved_docs, + query="Python", + ) + assert len(compressed_docs) < len(docs) From 67e0d222d10113470457fd4b0dc346d1c314b1d5 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 25 Nov 2025 11:54:34 +0000 Subject: [PATCH 05/93] fix: model in agents via api (#2174) --- .../api/answer/services/stream_processor.py | 17 ++++++++++++++--- application/core/model_configs.py | 6 +++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/application/api/answer/services/stream_processor.py b/application/api/answer/services/stream_processor.py index 5d97fbe8..912aff65 100644 --- a/application/api/answer/services/stream_processor.py +++ b/application/api/answer/services/stream_processor.py @@ -103,11 +103,10 @@ class StreamProcessor: def initialize(self): """Initialize all required components for processing""" - self._validate_and_set_model() self._configure_agent() + self._validate_and_set_model() self._configure_source() self._configure_retriever() - self._configure_agent() self._load_conversation_history() self._process_attachments() @@ -230,7 +229,12 @@ class StreamProcessor: ) self.model_id = requested_model else: - self.model_id = get_default_model_id() + # Check if agent has a default model configured + agent_default_model = self.agent_config.get("default_model_id", "") + if agent_default_model and validate_model_id(agent_default_model): + self.model_id = agent_default_model + else: + self.model_id = get_default_model_id() def _get_agent_key(self, agent_id: Optional[str], user_id: Optional[str]) -> tuple: """Get API key for agent with access control""" @@ -303,6 +307,10 @@ class StreamProcessor: data["sources"] = sources_list else: data["sources"] = [] + + # Preserve model configuration from agent + data["default_model_id"] = data.get("default_model_id", "") + return data def _configure_source(self): @@ -355,6 +363,7 @@ class StreamProcessor: "agent_type": data_key.get("agent_type", settings.AGENT_NAME), "user_api_key": api_key, "json_schema": data_key.get("json_schema"), + "default_model_id": data_key.get("default_model_id", ""), } ) self.initial_user_id = data_key.get("user") @@ -379,6 +388,7 @@ class StreamProcessor: "agent_type": data_key.get("agent_type", settings.AGENT_NAME), "user_api_key": self.agent_key, "json_schema": data_key.get("json_schema"), + "default_model_id": data_key.get("default_model_id", ""), } ) self.decoded_token = ( @@ -405,6 +415,7 @@ class StreamProcessor: "agent_type": settings.AGENT_NAME, "user_api_key": None, "json_schema": None, + "default_model_id": "", } ) diff --git a/application/core/model_configs.py b/application/core/model_configs.py index 5f75bc83..951abfb6 100644 --- a/application/core/model_configs.py +++ b/application/core/model_configs.py @@ -37,7 +37,7 @@ OPENAI_MODELS = [ supports_tools=True, supports_structured_output=True, supported_attachment_types=OPENAI_ATTACHMENTS, - context_window=400000, + context_window=200000, ), ), AvailableModel( @@ -49,7 +49,7 @@ OPENAI_MODELS = [ supports_tools=True, supports_structured_output=True, supported_attachment_types=OPENAI_ATTACHMENTS, - context_window=400000, + context_window=200000, ), ) ] @@ -133,7 +133,7 @@ GOOGLE_MODELS = [ supports_tools=True, supports_structured_output=True, supported_attachment_types=GOOGLE_ATTACHMENTS, - context_window=20000, # Set low for testing compression + context_window=2000000, ), ), ] From dc2faf7a7ee1ed0ad78fa9c57754a6f030b21e64 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 25 Nov 2025 14:08:22 +0000 Subject: [PATCH 06/93] fix: webhooks (#2175) --- application/worker.py | 67 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/application/worker.py b/application/worker.py index 3f957527..5c5dc367 100755 --- a/application/worker.py +++ b/application/worker.py @@ -146,6 +146,14 @@ def upload_index(full_path, file_data): def run_agent_logic(agent_config, input_data): try: + from application.core.model_utils import ( + get_api_key_for_provider, + get_default_model_id, + get_provider_from_model_id, + validate_model_id, + ) + from application.utils import calculate_doc_token_budget + source = agent_config.get("source") retriever = agent_config.get("retriever", "classic") if isinstance(source, DBRef): @@ -160,31 +168,62 @@ def run_agent_logic(agent_config, input_data): user_api_key = agent_config["key"] agent_type = agent_config.get("agent_type", "classic") decoded_token = {"sub": agent_config.get("user")} + json_schema = agent_config.get("json_schema") prompt = get_prompt(prompt_id, db["prompts"]) - agent = AgentCreator.create_agent( - agent_type, - endpoint="webhook", - llm_name=settings.LLM_PROVIDER, - model_id=settings.LLM_NAME, - api_key=settings.API_KEY, - user_api_key=user_api_key, - prompt=prompt, - chat_history=[], - decoded_token=decoded_token, - attachments=[], + + # Determine model_id: check agent's default_model_id, fallback to system default + agent_default_model = agent_config.get("default_model_id", "") + if agent_default_model and validate_model_id(agent_default_model): + model_id = agent_default_model + else: + model_id = get_default_model_id() + + # Get provider and API key for the selected model + provider = get_provider_from_model_id(model_id) if model_id else settings.LLM_PROVIDER + system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER) + + # Calculate proper doc_token_limit based on model's context window + history_token_limit = 2000 # Default for webhooks + doc_token_limit = calculate_doc_token_budget( + model_id=model_id, history_token_limit=history_token_limit ) + retriever = RetrieverCreator.create_retriever( retriever, source=source, chat_history=[], prompt=prompt, chunks=chunks, - token_limit=settings.DEFAULT_MAX_HISTORY, - model_id=settings.LLM_NAME, + doc_token_limit=doc_token_limit, + model_id=model_id, user_api_key=user_api_key, decoded_token=decoded_token, ) - answer = agent.gen(query=input_data, retriever=retriever) + + # Pre-fetch documents using the retriever + retrieved_docs = [] + try: + docs = retriever.search(input_data) + if docs: + retrieved_docs = docs + except Exception as e: + logging.warning(f"Failed to retrieve documents: {e}") + + agent = AgentCreator.create_agent( + agent_type, + endpoint="webhook", + llm_name=provider or settings.LLM_PROVIDER, + model_id=model_id, + api_key=system_api_key, + user_api_key=user_api_key, + prompt=prompt, + chat_history=[], + retrieved_docs=retrieved_docs, + decoded_token=decoded_token, + attachments=[], + json_schema=json_schema, + ) + answer = agent.gen(query=input_data) response_full = "" thought = "" source_log_docs = [] From 899b30da5e611b24385e38645a1bc9761edc8773 Mon Sep 17 00:00:00 2001 From: JustACodeA Date: Wed, 26 Nov 2025 11:52:23 +0100 Subject: [PATCH 07/93] feat: add German translation (#2170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds complete German (Deutsch) language support to DocsGPT. Changes: - Add de.json with full German translations - Register German in i18n configuration - Add German to language selector dropdown 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude --- frontend/src/locale/de.json | 646 ++++++++++++++++++++++++++++++ frontend/src/locale/i18n.ts | 4 + frontend/src/settings/General.tsx | 1 + 3 files changed, 651 insertions(+) create mode 100644 frontend/src/locale/de.json diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json new file mode 100644 index 00000000..a4b10003 --- /dev/null +++ b/frontend/src/locale/de.json @@ -0,0 +1,646 @@ +{ + "language": "Deutsch", + "chat": "Chat", + "chats": "Chats", + "newChat": "Neuer Chat", + "inputPlaceholder": "Wie kann DocsGPT dir helfen?", + "tagline": "DocsGPT verwendet GenAI, bitte überprüfe kritische Informationen anhand der Quellen.", + "sourceDocs": "Quelle", + "none": "Keine", + "cancel": "Abbrechen", + "help": "Hilfe", + "emailUs": "E-Mail senden", + "documentation": "Dokumentation", + "manageAgents": "Agenten verwalten", + "demo": [ + { + "header": "Über DocsGPT lernen", + "query": "Was ist DocsGPT?" + }, + { + "header": "Dokumentation zusammenfassen", + "query": "Fasse den aktuellen Kontext zusammen" + }, + { + "header": "Code schreiben", + "query": "Schreibe Code für eine API-Anfrage an /api/answer" + }, + { + "header": "Lernunterstützung", + "query": "Schreibe mögliche Fragen zum Kontext" + } + ], + "settings": { + "label": "Einstellungen", + "general": { + "label": "Allgemein", + "selectTheme": "Design auswählen", + "light": "Hell", + "dark": "Dunkel", + "selectLanguage": "Sprache auswählen", + "chunks": "Chunks pro Anfrage", + "prompt": "Aktiver Prompt", + "deleteAllLabel": "Alle Konversationen löschen", + "deleteAllBtn": "Alle löschen", + "addNew": "Neu hinzufügen", + "convHistory": "Konversationsverlauf", + "none": "Keine", + "low": "Niedrig", + "medium": "Mittel", + "high": "Hoch", + "unlimited": "Unbegrenzt", + "default": "Standard", + "add": "Hinzufügen" + }, + "sources": { + "title": "Hier kannst du alle verfügbaren Quelldateien verwalten, die dir zur Verfügung stehen und die du hochgeladen hast.", + "label": "Quellen", + "name": "Quellenname", + "date": "Vektor-Datum", + "type": "Typ", + "tokenUsage": "Token-Verbrauch", + "noData": "Keine vorhandenen Quellen", + "searchPlaceholder": "Suchen...", + "addNew": "Neu hinzufügen", + "addSource": "Quelle hinzufügen", + "addChunk": "Chunk hinzufügen", + "preLoaded": "Vorgeladen", + "private": "Privat", + "sync": "Synchronisieren", + "syncing": "Synchronisiere...", + "syncConfirmation": "Bist du sicher, dass du \"{{sourceName}}\" synchronisieren möchtest? Dies aktualisiert den Inhalt mit deinem Cloud-Speicher und kann Änderungen an einzelnen Chunks überschreiben.", + "syncFrequency": { + "never": "Nie", + "daily": "Täglich", + "weekly": "Wöchentlich", + "monthly": "Monatlich" + }, + "actions": "Aktionen", + "view": "Anzeigen", + "deleteWarning": "Bist du sicher, dass du \"{{name}}\" löschen möchtest?", + "confirmDelete": "Bist du sicher, dass du diese Datei löschen möchtest? Diese Aktion kann nicht rückgängig gemacht werden.", + "backToAll": "Zurück zu allen Quellen", + "chunks": "Chunks", + "noChunks": "Keine Chunks gefunden", + "noChunksAlt": "Keine Chunks gefunden", + "goToSources": "Zu den Quellen", + "uploadNew": "Neu hochladen", + "searchFiles": "Dateien suchen...", + "noResults": "Keine Ergebnisse gefunden", + "fileName": "Name", + "tokens": "Tokens", + "size": "Größe", + "fileAlt": "Datei", + "folderAlt": "Ordner", + "parentFolderAlt": "Übergeordneter Ordner", + "menuAlt": "Menü", + "tokensUnit": "Tokens", + "editAlt": "Bearbeiten", + "uploading": "Wird hochgeladen…", + "deleting": "Wird gelöscht…", + "queued": "In Warteschlange: {{count}}", + "addFile": "Datei hinzufügen", + "uploadingFilesTitle": "Dateien werden hochgeladen...", + "deletingTitle": "Wird gelöscht...", + "deleteDirectoryWarning": "Bist du sicher, dass du das Verzeichnis \"{{name}}\" und seinen gesamten Inhalt löschen möchtest? Diese Aktion kann nicht rückgängig gemacht werden.", + "searchAlt": "Suchen" + }, + "apiKeys": { + "label": "Chatbots", + "name": "Name", + "key": "API-Schlüssel", + "sourceDoc": "Quelldokument", + "createNew": "Neu erstellen", + "noData": "Keine vorhandenen Chatbots", + "deleteConfirmation": "Bist du sicher, dass du den API-Schlüssel '{{name}}' löschen möchtest?", + "description": "Hier kannst du deine Chatbots erstellen und verwalten. Chatbots können als Widgets auf Websites eingebunden oder in deinen Anwendungen verwendet werden." + }, + "analytics": { + "label": "Analytik", + "filterByChatbot": "Nach Chatbot filtern", + "selectChatbot": "Chatbot auswählen", + "filterOptions": { + "hour": "Stunde", + "last24Hours": "24 Stunden", + "last7Days": "7 Tage", + "last15Days": "15 Tage", + "last30Days": "30 Tage" + }, + "messages": "Nachrichten", + "tokenUsage": "Token-Verbrauch", + "userFeedback": "Benutzer-Feedback", + "filterPlaceholder": "Filter", + "none": "Keine", + "positiveFeedback": "Positives Feedback", + "negativeFeedback": "Negatives Feedback" + }, + "logs": { + "label": "Protokolle", + "filterByChatbot": "Nach Chatbot filtern", + "selectChatbot": "Chatbot auswählen", + "none": "Keine", + "tableHeader": "API-generierte / Chatbot-Konversationen" + }, + "tools": { + "label": "Werkzeuge", + "searchPlaceholder": "Werkzeuge suchen...", + "addTool": "Werkzeug hinzufügen", + "noToolsFound": "Keine Werkzeuge gefunden", + "selectToolSetup": "Wähle ein Werkzeug zur Einrichtung", + "settingsIconAlt": "Einstellungssymbol", + "configureToolAria": "{{toolName}} konfigurieren", + "toggleToolAria": "{{toolName}} umschalten", + "manageTools": "Zu den Werkzeugen", + "edit": "Bearbeiten", + "delete": "Löschen", + "deleteWarning": "Bist du sicher, dass du das Werkzeug \"{{toolName}}\" löschen möchtest?", + "unsavedChanges": "Du hast ungespeicherte Änderungen, die verloren gehen, wenn du ohne Speichern verlässt.", + "leaveWithoutSaving": "Ohne Speichern verlassen", + "saveAndLeave": "Speichern und verlassen", + "customName": "Benutzerdefinierter Name", + "customNamePlaceholder": "Gib einen benutzerdefinierten Namen ein (optional)", + "authentication": "Authentifizierung", + "actions": "Aktionen", + "addAction": "Aktion hinzufügen", + "noActionsFound": "Keine Aktionen gefunden", + "url": "URL", + "urlPlaceholder": "URL eingeben", + "method": "Methode", + "description": "Beschreibung", + "descriptionPlaceholder": "Beschreibung eingeben", + "headers": "Header", + "queryParameters": "Abfrageparameter", + "body": "Body", + "deleteActionWarning": "Bist du sicher, dass du die Aktion \"{{name}}\" löschen möchtest?", + "backToAllTools": "Zurück zu allen Werkzeugen", + "save": "Speichern", + "fieldName": "Feldname", + "fieldType": "Feldtyp", + "filledByLLM": "Vom LLM ausgefüllt", + "fieldDescription": "Feldbeschreibung", + "value": "Wert", + "addProperty": "Eigenschaft hinzufügen", + "propertyName": "Neuer Eigenschaftsschlüssel", + "add": "Hinzufügen", + "cancel": "Abbrechen", + "addNew": "Neu hinzufügen", + "name": "Name", + "type": "Typ", + "mcp": { + "addServer": "MCP-Server hinzufügen", + "editServer": "Server bearbeiten", + "serverName": "Servername", + "serverUrl": "Server-URL", + "headerName": "Header-Name", + "timeout": "Timeout (Sekunden)", + "testConnection": "Verbindung testen", + "testing": "Teste...", + "saving": "Speichere...", + "save": "Speichern", + "cancel": "Abbrechen", + "noAuth": "Keine Authentifizierung", + "oauthInProgress": "Warte auf OAuth-Abschluss...", + "oauthCompleted": "OAuth erfolgreich abgeschlossen", + "authType": "Authentifizierungstyp", + "defaultServerName": "Mein MCP-Server", + "authTypes": { + "none": "Keine Authentifizierung", + "apiKey": "API-Schlüssel", + "bearer": "Bearer-Token", + "oauth": "OAuth", + "basic": "Basis-Authentifizierung" + }, + "placeholders": { + "serverUrl": "https://api.beispiel.com", + "apiKey": "Dein geheimer API-Schlüssel", + "bearerToken": "Dein geheimes Token", + "username": "Dein Benutzername", + "password": "Dein Passwort", + "oauthScopes": "OAuth-Bereiche (kommagetrennt)" + }, + "errors": { + "nameRequired": "Servername ist erforderlich", + "urlRequired": "Server-URL ist erforderlich", + "invalidUrl": "Bitte gib eine gültige URL ein", + "apiKeyRequired": "API-Schlüssel ist erforderlich", + "tokenRequired": "Bearer-Token ist erforderlich", + "usernameRequired": "Benutzername ist erforderlich", + "passwordRequired": "Passwort ist erforderlich", + "testFailed": "Verbindungstest fehlgeschlagen", + "saveFailed": "MCP-Server konnte nicht gespeichert werden", + "oauthFailed": "OAuth-Prozess fehlgeschlagen oder abgebrochen", + "oauthTimeout": "OAuth-Prozess abgelaufen, bitte erneut versuchen", + "timeoutRange": "Timeout muss zwischen 1 und 300 Sekunden liegen" + } + } + }, + "scrollTabsLeft": "Tabs nach links scrollen", + "tabsAriaLabel": "Einstellungs-Tabs", + "scrollTabsRight": "Tabs nach rechts scrollen" + }, + "modals": { + "uploadDoc": { + "label": "Neues Dokument hochladen", + "select": "Wähle, wie du dein Dokument zu DocsGPT hochladen möchtest", + "selectSource": "Wähle die Art, wie du deine Quelle hinzufügen möchtest", + "selectedFiles": "Ausgewählte Dateien", + "noFilesSelected": "Keine Dateien ausgewählt", + "file": "Vom Gerät hochladen", + "back": "Zurück", + "wait": "Bitte warten ...", + "remote": "Von Website sammeln", + "start": "Chat starten", + "name": "Name", + "choose": "Dateien auswählen", + "info": "Bitte lade .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip hoch (max. 25 MB)", + "uploadedFiles": "Hochgeladene Dateien", + "cancel": "Abbrechen", + "train": "Trainieren", + "link": "Link", + "urlLink": "URL-Link", + "repoUrl": "Repository-URL", + "reddit": { + "id": "Client-ID", + "secret": "Client-Secret", + "agent": "User-Agent", + "searchQueries": "Suchanfragen", + "numberOfPosts": "Anzahl der Beiträge", + "addQuery": "Anfrage hinzufügen" + }, + "drag": { + "title": "Anhänge hier ablegen", + "description": "Loslassen, um deine Anhänge hochzuladen" + }, + "progress": { + "upload": "Upload läuft", + "training": "Upload läuft", + "completed": "Upload abgeschlossen", + "failed": "Upload fehlgeschlagen", + "wait": "Dies kann einige Minuten dauern", + "preparing": "Upload wird vorbereitet", + "tokenLimit": "Token-Limit überschritten, bitte lade ein kleineres Dokument hoch", + "expandDetails": "Upload-Details erweitern", + "collapseDetails": "Upload-Details einklappen", + "dismiss": "Upload-Benachrichtigung schließen", + "uploadProgress": "Upload-Fortschritt {{progress}}%", + "clear": "Löschen" + }, + "showAdvanced": "Erweiterte Optionen anzeigen", + "hideAdvanced": "Erweiterte Optionen ausblenden", + "ingestors": { + "local_file": { + "label": "Datei hochladen", + "heading": "Neues Dokument hochladen" + }, + "crawler": { + "label": "Crawler", + "heading": "Inhalt mit Web-Crawler hinzufügen" + }, + "url": { + "label": "Link", + "heading": "Inhalt von URL hinzufügen" + }, + "github": { + "label": "GitHub", + "heading": "Inhalt von GitHub hinzufügen" + }, + "reddit": { + "label": "Reddit", + "heading": "Inhalt von Reddit hinzufügen" + }, + "google_drive": { + "label": "Google Drive", + "heading": "Von Google Drive hochladen" + } + }, + "connectors": { + "auth": { + "connectedUser": "Verbundener Benutzer", + "authFailed": "Authentifizierung fehlgeschlagen", + "authUrlFailed": "Autorisierungs-URL konnte nicht abgerufen werden", + "popupBlocked": "Authentifizierungsfenster konnte nicht geöffnet werden. Bitte erlaube Popups.", + "authCancelled": "Authentifizierung wurde abgebrochen", + "connectedAs": "Verbunden als {{email}}", + "disconnect": "Trennen" + }, + "googleDrive": { + "connect": "Mit Google Drive verbinden", + "sessionExpired": "Sitzung abgelaufen. Bitte verbinde dich erneut mit Google Drive.", + "sessionExpiredGeneric": "Sitzung abgelaufen. Bitte verbinde dein Konto erneut.", + "validateFailed": "Sitzung konnte nicht validiert werden. Bitte verbinde dich erneut.", + "noSession": "Keine gültige Sitzung gefunden. Bitte verbinde dich erneut mit Google Drive.", + "noAccessToken": "Kein Zugriffstoken verfügbar. Bitte verbinde dich erneut mit Google Drive.", + "pickerFailed": "Dateiauswahl konnte nicht geöffnet werden. Bitte versuche es erneut.", + "selectedFiles": "Ausgewählte Dateien", + "selectFiles": "Dateien auswählen", + "loading": "Laden...", + "noFilesSelected": "Keine Dateien oder Ordner ausgewählt", + "folders": "Ordner", + "files": "Dateien", + "remove": "Entfernen", + "folderAlt": "Ordner", + "fileAlt": "Datei" + } + } + }, + "createAPIKey": { + "label": "Neuen API-Schlüssel erstellen", + "apiKeyName": "API-Schlüssel-Name", + "chunks": "Chunks pro Anfrage", + "prompt": "Aktiven Prompt auswählen", + "sourceDoc": "Quelldokument", + "create": "Erstellen" + }, + "saveKey": { + "note": "Bitte speichere deinen Schlüssel", + "disclaimer": "Dies ist das einzige Mal, dass dein Schlüssel angezeigt wird.", + "copy": "Kopieren", + "copied": "Kopiert", + "confirm": "Ich habe den Schlüssel gespeichert", + "apiKeyLabel": "API-Schlüssel" + }, + "deleteConv": { + "confirm": "Bist du sicher, dass du alle Konversationen löschen möchtest?", + "delete": "Löschen" + }, + "shareConv": { + "label": "Öffentliche Seite zum Teilen erstellen", + "note": "Quelldokument, persönliche Informationen und weitere Konversationen bleiben privat", + "create": "Erstellen", + "option": "Benutzern weitere Eingaben erlauben" + }, + "configTool": { + "title": "Werkzeug-Konfiguration", + "type": "Typ", + "apiKeyLabel": "API-Schlüssel / OAuth", + "apiKeyPlaceholder": "API-Schlüssel / OAuth eingeben", + "addButton": "Werkzeug hinzufügen", + "closeButton": "Schließen", + "customNamePlaceholder": "Benutzerdefinierten Namen eingeben (optional)" + }, + "prompts": { + "addPrompt": "Prompt hinzufügen", + "addDescription": "Füge deinen benutzerdefinierten Prompt hinzu und speichere ihn in DocsGPT", + "editPrompt": "Prompt bearbeiten", + "editDescription": "Bearbeite deinen benutzerdefinierten Prompt und speichere ihn in DocsGPT", + "promptName": "Prompt-Name", + "promptText": "Prompt-Text", + "save": "Speichern", + "cancel": "Abbrechen", + "nameExists": "Name existiert bereits", + "deleteConfirmation": "Bist du sicher, dass du den Prompt '{{name}}' löschen möchtest?", + "placeholderText": "Gib hier deinen Prompt-Text ein...", + "addExamplePlaceholder": "Bitte fasse diesen Text zusammen:", + "variablesLabel": "Variablen", + "variablesSubtext": "Klicken zum Einfügen in den Prompt", + "variablesDescription": "Klicken zum Einfügen in den Prompt", + "systemVariables": "Klicken zum Einfügen in den Prompt", + "toolVariables": "Werkzeug-Variablen", + "systemVariablesDropdownLabel": "System-Variablen", + "systemVariableOptions": { + "sourceContent": "Quelleninhalte", + "sourceSummaries": "Alias für Inhalte (abwärtskompatibel)", + "sourceDocuments": "Dokumentenobjekte-Liste", + "sourceCount": "Anzahl der abgerufenen Dokumente", + "systemDate": "Aktuelles Datum (JJJJ-MM-TT)", + "systemTime": "Aktuelle Uhrzeit (HH:MM:SS)", + "systemTimestamp": "ISO 8601 Zeitstempel", + "systemRequestId": "Eindeutige Anfrage-ID", + "systemUserId": "Aktuelle Benutzer-ID" + }, + "learnAboutPrompts": "Mehr über Prompts erfahren →", + "publicPromptEditDisabled": "Öffentliche Prompts können nicht bearbeitet werden", + "promptTypePublic": "öffentlich", + "promptTypePrivate": "privat" + }, + "chunk": { + "add": "Chunk hinzufügen", + "edit": "Bearbeiten", + "title": "Titel", + "enterTitle": "Titel eingeben", + "bodyText": "Textkörper", + "promptText": "Prompt-Text", + "save": "Speichern", + "close": "Schließen", + "cancel": "Abbrechen", + "delete": "Löschen", + "deleteConfirmation": "Bist du sicher, dass du diesen Chunk löschen möchtest?" + }, + "addAction": { + "title": "Neue Aktion", + "actionNamePlaceholder": "Aktionsname", + "invalidFormat": "Ungültiges Funktionsnamenformat. Verwende nur Buchstaben, Zahlen, Unterstriche und Bindestriche.", + "formatHelp": "Verwende nur Buchstaben, Zahlen, Unterstriche und Bindestriche (z.B. `get_data`, `send_report`, etc.)", + "addButton": "Hinzufügen" + }, + "agentDetails": { + "title": "Zugangsdaten", + "publicLink": "Öffentlicher Link", + "apiKey": "API-Schlüssel", + "webhookUrl": "Webhook-URL", + "generate": "Generieren", + "test": "Testen", + "learnMore": "Mehr erfahren" + } + }, + "sharedConv": { + "subtitle": "Erstellt mit", + "button": "Mit DocsGPT starten", + "meta": "DocsGPT verwendet GenAI, bitte überprüfe kritische Informationen anhand der Quellen." + }, + "convTile": { + "share": "Teilen", + "delete": "Löschen", + "rename": "Umbenennen", + "deleteWarning": "Bist du sicher, dass du diese Konversation löschen möchtest?" + }, + "pagination": { + "rowsPerPage": "Zeilen pro Seite", + "pageOf": "Seite {{currentPage}} von {{totalPages}}", + "firstPage": "Erste Seite", + "previousPage": "Vorherige Seite", + "nextPage": "Nächste Seite", + "lastPage": "Letzte Seite" + }, + "conversation": { + "copy": "Kopieren", + "copied": "Kopiert", + "speak": "Vorlesen", + "answer": "Antwort", + "edit": { + "update": "Aktualisieren", + "cancel": "Abbrechen", + "placeholder": "Aktualisierte Anfrage eingeben..." + }, + "sources": { + "title": "Quellen", + "text": "Wähle deine Quellen", + "link": "Quellen-Link", + "view_more": "{{count}} weitere Quellen", + "noSourcesAvailable": "Keine Quellen verfügbar" + }, + "attachments": { + "attach": "Anhängen", + "remove": "Anhang entfernen" + }, + "retry": "Erneut versuchen", + "reasoning": "Begründung" + }, + "agents": { + "title": "Agenten", + "description": "Entdecke und erstelle benutzerdefinierte Versionen von DocsGPT, die Anweisungen, zusätzliches Wissen und beliebige Kombinationen von Fähigkeiten kombinieren", + "newAgent": "Neuer Agent", + "backToAll": "Zurück zu allen Agenten", + "sections": { + "template": { + "title": "Von DocsGPT", + "description": "Von DocsGPT bereitgestellte Agenten", + "emptyState": "Keine Vorlagen-Agenten gefunden." + }, + "user": { + "title": "Von mir", + "description": "Von dir erstellte oder veröffentlichte Agenten", + "emptyState": "Du hast noch keine Agenten erstellt." + }, + "shared": { + "title": "Mit mir geteilt", + "description": "Über einen öffentlichen Link importierte Agenten", + "emptyState": "Keine geteilten Agenten gefunden." + } + }, + "form": { + "headings": { + "new": "Neuer Agent", + "edit": "Agent bearbeiten", + "draft": "Neuer Agent (Entwurf)" + }, + "buttons": { + "publish": "Veröffentlichen", + "save": "Speichern", + "saveDraft": "Entwurf speichern", + "cancel": "Abbrechen", + "delete": "Löschen", + "logs": "Protokolle", + "accessDetails": "Zugangsdaten", + "add": "Hinzufügen" + }, + "sections": { + "meta": "Meta", + "source": "Quelle", + "prompt": "Prompt", + "tools": "Werkzeuge", + "agentType": "Agententyp", + "models": "Modelle", + "advanced": "Erweitert", + "preview": "Vorschau" + }, + "placeholders": { + "agentName": "Agentenname", + "describeAgent": "Beschreibe deinen Agenten", + "selectSources": "Quellen auswählen", + "chunksPerQuery": "Chunks pro Anfrage", + "selectType": "Typ auswählen", + "selectTools": "Werkzeuge auswählen", + "selectModels": "Modelle für diesen Agenten auswählen", + "selectDefaultModel": "Standardmodell auswählen", + "enterTokenLimit": "Token-Limit eingeben", + "enterRequestLimit": "Anfrage-Limit eingeben" + }, + "sourcePopup": { + "title": "Quellen auswählen", + "searchPlaceholder": "Quellen suchen...", + "noOptionsMessage": "Keine Quellen verfügbar" + }, + "toolsPopup": { + "title": "Werkzeuge auswählen", + "searchPlaceholder": "Werkzeuge suchen...", + "noOptionsMessage": "Keine Werkzeuge verfügbar" + }, + "modelsPopup": { + "title": "Modelle auswählen", + "searchPlaceholder": "Modelle suchen...", + "noOptionsMessage": "Keine Modelle verfügbar" + }, + "upload": { + "clickToUpload": "Klicken zum Hochladen", + "dragAndDrop": " oder per Drag & Drop" + }, + "agentTypes": { + "classic": "Klassisch", + "react": "ReAct" + }, + "labels": { + "defaultModel": "Standardmodell" + }, + "advanced": { + "jsonSchema": "JSON-Antwortschema", + "jsonSchemaDescription": "Definiere ein JSON-Schema, um ein strukturiertes Ausgabeformat zu erzwingen", + "validJson": "Gültiges JSON", + "invalidJson": "Ungültiges JSON - zur Aktivierung des Speicherns beheben", + "tokenLimiting": "Token-Limitierung", + "tokenLimitingDescription": "Begrenze die täglich von diesem Agenten verwendbaren Tokens", + "requestLimiting": "Anfrage-Limitierung", + "requestLimitingDescription": "Begrenze die täglich an diesen Agenten gestellten Anfragen" + }, + "preview": { + "publishedPreview": "Veröffentlichte Agenten können hier in der Vorschau angezeigt werden" + }, + "externalKb": "Externe KB" + }, + "logs": { + "title": "Agenten-Protokolle", + "lastUsedAt": "Zuletzt verwendet am", + "noUsageHistory": "Kein Nutzungsverlauf", + "tableHeader": "Agenten-Endpunkt-Protokolle" + }, + "shared": { + "notFound": "Kein Agent gefunden. Bitte stelle sicher, dass der Agent geteilt ist." + }, + "preview": { + "testMessage": "Teste deinen Agenten hier. Veröffentlichte Agenten können in Konversationen verwendet werden." + }, + "deleteConfirmation": "Bist du sicher, dass du diesen Agenten löschen möchtest?" + }, + "components": { + "fileUpload": { + "clickToUpload": "Klicken zum Hochladen oder per Drag & Drop", + "dropFiles": "Dateien hier ablegen", + "fileTypes": "PNG, JPG, JPEG bis zu", + "sizeLimitUnit": "MB", + "fileSizeError": "Datei überschreitet {{size}}MB-Limit" + } + }, + "pageNotFound": { + "title": "404", + "message": "Die gesuchte Seite existiert nicht.", + "goHome": "Zur Startseite" + }, + "filePicker": { + "searchPlaceholder": "Dateien und Ordner suchen...", + "itemsSelected": "{{count}} ausgewählt", + "name": "Name", + "lastModified": "Zuletzt geändert", + "size": "Größe" + }, + "actionButtons": { + "openNewChat": "Neuen Chat öffnen", + "share": "Teilen" + }, + "mermaid": { + "downloadOptions": "Download-Optionen", + "viewCode": "Code anzeigen", + "decreaseZoom": "Verkleinern", + "resetZoom": "Zoom zurücksetzen", + "increaseZoom": "Vergrößern" + }, + "navigation": { + "agents": "Agenten" + }, + "notification": { + "ariaLabel": "Benachrichtigung", + "closeAriaLabel": "Benachrichtigung schließen" + }, + "prompts": { + "textAriaLabel": "Prompt-Text" + } +} diff --git a/frontend/src/locale/i18n.ts b/frontend/src/locale/i18n.ts index 4f2e2029..6edebd09 100644 --- a/frontend/src/locale/i18n.ts +++ b/frontend/src/locale/i18n.ts @@ -8,6 +8,7 @@ import jp from './jp.json'; //Japanese import zh from './zh.json'; //Mandarin import zhTW from './zh-TW.json'; //Traditional Chinese import ru from './ru.json'; //Russian +import de from './de.json'; //German i18n .use(LanguageDetector) @@ -32,6 +33,9 @@ i18n ru: { translation: ru, }, + de: { + translation: de, + }, }, fallbackLng: 'en', detection: { diff --git a/frontend/src/settings/General.tsx b/frontend/src/settings/General.tsx index d6c5bde0..442f7076 100644 --- a/frontend/src/settings/General.tsx +++ b/frontend/src/settings/General.tsx @@ -30,6 +30,7 @@ export default function General() { const languageOptions = [ { label: 'English', value: 'en' }, + { label: 'Deutsch', value: 'de' }, { label: 'Español', value: 'es' }, { label: '日本語', value: 'jp' }, { label: '普通话', value: 'zh' }, From 3352d424142dd5303dbff35bc3d4d89805c75464 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 26 Nov 2025 17:12:02 +0000 Subject: [PATCH 08/93] fix(frontend): use bracket notation for tool variable paths (#2176) --- frontend/src/preferences/PromptsModal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/preferences/PromptsModal.tsx b/frontend/src/preferences/PromptsModal.tsx index d6071687..b724e968 100644 --- a/frontend/src/preferences/PromptsModal.tsx +++ b/frontend/src/preferences/PromptsModal.tsx @@ -191,7 +191,7 @@ const useToolVariables = () => { } filteredActions.push({ label: `${action.name} (${tool.displayName || tool.name})`, - value: `tools.${toolIdentifier}.${action.name}`, + value: `tools['${toolIdentifier}'].${action.name}`, }); } } From 9b9f95710a2d7654694ed4ba1d98724586f9cae4 Mon Sep 17 00:00:00 2001 From: Siddhant Rai <47355538+siiddhantt@users.noreply.github.com> Date: Thu, 4 Dec 2025 21:16:37 +0530 Subject: [PATCH 09/93] feat: agent search functionality with filters and loading states (#2179) * feat: implement agent search functionality with filters and loading states * style: improve layout and styling of agent search input and description --- frontend/src/agents/AgentsList.tsx | 197 ++++++++++++++++---- frontend/src/agents/agents.config.ts | 12 +- frontend/src/agents/hooks/useAgentSearch.ts | 122 ++++++++++++ frontend/src/agents/hooks/useAgentsFetch.ts | 83 +++++++++ frontend/src/locale/de.json | 9 + frontend/src/locale/en.json | 9 + frontend/src/locale/es.json | 9 + frontend/src/locale/jp.json | 9 + frontend/src/locale/ru.json | 9 + frontend/src/locale/zh-TW.json | 9 + frontend/src/locale/zh.json | 9 + 11 files changed, 431 insertions(+), 46 deletions(-) create mode 100644 frontend/src/agents/hooks/useAgentSearch.ts create mode 100644 frontend/src/agents/hooks/useAgentsFetch.ts diff --git a/frontend/src/agents/AgentsList.tsx b/frontend/src/agents/AgentsList.tsx index bdccb0f1..0c3c1d0e 100644 --- a/frontend/src/agents/AgentsList.tsx +++ b/frontend/src/agents/AgentsList.tsx @@ -1,8 +1,9 @@ -import { useEffect, useState } from 'react'; +import { useEffect } from 'react'; import { useTranslation } from 'react-i18next'; import { useDispatch, useSelector } from 'react-redux'; import { useNavigate } from 'react-router-dom'; +import Search from '../assets/search.svg'; import Spinner from '../components/Spinner'; import { setConversation, @@ -10,19 +11,40 @@ import { } from '../conversation/conversationSlice'; import { selectSelectedAgent, - selectToken, setSelectedAgent, } from '../preferences/preferenceSlice'; import AgentCard from './AgentCard'; -import { agentSectionsConfig } from './agents.config'; +import { AgentSectionId, agentSectionsConfig } from './agents.config'; +import { AgentFilterTab, useAgentSearch } from './hooks/useAgentSearch'; +import { useAgentsFetch } from './hooks/useAgentsFetch'; import { Agent } from './types'; +const FILTER_TABS: { id: AgentFilterTab; labelKey: string }[] = [ + { id: 'all', labelKey: 'agents.filters.all' }, + { id: 'template', labelKey: 'agents.filters.byDocsGPT' }, + { id: 'user', labelKey: 'agents.filters.byMe' }, + { id: 'shared', labelKey: 'agents.filters.shared' }, +]; + export default function AgentsList() { const { t } = useTranslation(); const dispatch = useDispatch(); - const token = useSelector(selectToken); const selectedAgent = useSelector(selectSelectedAgent); + const { isLoading } = useAgentsFetch(); + + const { + searchQuery, + setSearchQuery, + activeFilter, + setActiveFilter, + filteredAgentsBySection, + totalAgentsBySection, + hasAnyAgents, + hasFilteredResults, + isDataLoaded, + } = useAgentSearch(); + useEffect(() => { dispatch(setConversation([])); dispatch( @@ -31,57 +53,150 @@ export default function AgentsList() { }), ); if (selectedAgent) dispatch(setSelectedAgent(null)); - }, [token]); + }, []); + + const visibleSections = agentSectionsConfig.filter((config) => { + if (activeFilter !== 'all') { + return config.id === activeFilter; + } + const sectionId = config.id as AgentSectionId; + const hasAgentsInSection = totalAgentsBySection[sectionId] > 0; + const hasFilteredAgents = filteredAgentsBySection[sectionId].length > 0; + const sectionDataLoaded = isDataLoaded[sectionId]; + + if (!sectionDataLoaded) return true; + if (searchQuery) return hasFilteredAgents; + if (config.id === 'user') return true; + return hasAgentsInSection; + }); + + const showSearchEmptyState = + searchQuery && + hasAnyAgents && + !hasFilteredResults && + activeFilter === 'all'; + return (

{t('agents.title')}

-

+

{t('agents.description')}

- {agentSectionsConfig.map((sectionConfig) => ( - + +
+
+ + setSearchQuery(e.target.value)} + placeholder={t('agents.searchPlaceholder')} + className="h-[44px] w-full rounded-full border border-[#E5E5E5] bg-white py-2 pr-5 pl-11 text-sm shadow-[0_1px_4px_rgba(0,0,0,0.06)] transition-shadow outline-none placeholder:text-[#9CA3AF] focus:shadow-[0_2px_8px_rgba(0,0,0,0.1)] dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white dark:shadow-none dark:placeholder:text-[#6B7280]" + /> +
+ +
+ {FILTER_TABS.map((tab) => ( + + ))} +
+
+ + {visibleSections.map((sectionConfig) => ( + ))} + + {showSearchEmptyState && ( +
+

{t('agents.noSearchResults')}

+

{t('agents.tryDifferentSearch')}

+
+ )}
); } +interface AgentSectionProps { + config: (typeof agentSectionsConfig)[number]; + filteredAgents: Agent[]; + totalAgents: number; + searchQuery: string; + isFilteredView: boolean; + isLoading: boolean; +} + function AgentSection({ config, -}: { - config: (typeof agentSectionsConfig)[number]; -}) { + filteredAgents, + totalAgents, + searchQuery, + isFilteredView, + isLoading, +}: AgentSectionProps) { const { t } = useTranslation(); const navigate = useNavigate(); const dispatch = useDispatch(); - const token = useSelector(selectToken); - const agents = useSelector(config.selectData); - - const [loading, setLoading] = useState(true); + const allAgents = useSelector(config.selectData); const updateAgents = (updatedAgents: Agent[]) => { dispatch(config.updateAction(updatedAgents)); }; - useEffect(() => { - const getAgents = async () => { - setLoading(true); - try { - const response = await config.fetchAgents(token); - if (!response.ok) - throw new Error(`Failed to fetch ${config.id} agents`); - const data = await response.json(); - dispatch(config.updateAction(data)); - } catch (error) { - console.error(`Error fetching ${config.id} agents:`, error); - dispatch(config.updateAction([])); - } finally { - setLoading(false); - } - }; - getAgents(); - }, [token, config, dispatch]); + const hasNoAgentsAtAll = !isLoading && totalAgents === 0; + const isSearchingWithNoResults = + !isLoading && searchQuery && filteredAgents.length === 0 && totalAgents > 0; + + if (isFilteredView && isSearchingWithNoResults) { + return ( +
+

{t('agents.noSearchResults')}

+

{t('agents.tryDifferentSearch')}

+
+ ); + } + + if (isFilteredView && hasNoAgentsAtAll) { + return ( +
+

{t(`agents.sections.${config.id}.emptyState`)}

+ {config.showNewAgentButton && ( + + )} +
+ ); + } + return (
@@ -103,24 +218,24 @@ function AgentSection({ )}
- {loading ? ( -
+ {isLoading ? ( +
- ) : agents && agents.length > 0 ? ( + ) : filteredAgents.length > 0 ? (
- {agents.map((agent) => ( + {filteredAgents.map((agent) => ( ))}
- ) : ( -
+ ) : hasNoAgentsAtAll ? ( +

{t(`agents.sections.${config.id}.emptyState`)}

{config.showNewAgentButton && (
- )} + ) : null}
); diff --git a/frontend/src/agents/agents.config.ts b/frontend/src/agents/agents.config.ts index b6be5015..3569600e 100644 --- a/frontend/src/agents/agents.config.ts +++ b/frontend/src/agents/agents.config.ts @@ -1,16 +1,18 @@ import userService from '../api/services/userService'; import { selectAgents, - selectTemplateAgents, selectSharedAgents, + selectTemplateAgents, setAgents, - setTemplateAgents, setSharedAgents, + setTemplateAgents, } from '../preferences/preferenceSlice'; +export type AgentSectionId = 'template' | 'user' | 'shared'; + export const agentSectionsConfig = [ { - id: 'template', + id: 'template' as const, title: 'By DocsGPT', description: 'Agents provided by DocsGPT', showNewAgentButton: false, @@ -20,7 +22,7 @@ export const agentSectionsConfig = [ updateAction: setTemplateAgents, }, { - id: 'user', + id: 'user' as const, title: 'By me', description: 'Agents created or published by you', showNewAgentButton: true, @@ -30,7 +32,7 @@ export const agentSectionsConfig = [ updateAction: setAgents, }, { - id: 'shared', + id: 'shared' as const, title: 'Shared with me', description: 'Agents imported by using a public link', showNewAgentButton: false, diff --git a/frontend/src/agents/hooks/useAgentSearch.ts b/frontend/src/agents/hooks/useAgentSearch.ts new file mode 100644 index 00000000..7a5edb9e --- /dev/null +++ b/frontend/src/agents/hooks/useAgentSearch.ts @@ -0,0 +1,122 @@ +import { useCallback, useMemo, useState } from 'react'; +import { useSelector } from 'react-redux'; + +import { + selectAgents, + selectSharedAgents, + selectTemplateAgents, +} from '../../preferences/preferenceSlice'; +import { AgentSectionId } from '../agents.config'; +import { Agent } from '../types'; + +export type AgentFilterTab = 'all' | AgentSectionId; + +export type AgentsBySection = Record; + +interface UseAgentSearchResult { + searchQuery: string; + setSearchQuery: (query: string) => void; + activeFilter: AgentFilterTab; + setActiveFilter: (filter: AgentFilterTab) => void; + filteredAgentsBySection: AgentsBySection; + totalAgentsBySection: Record; + hasAnyAgents: boolean; + hasFilteredResults: boolean; + isDataLoaded: Record; +} + +const filterAgentsByQuery = ( + agents: Agent[] | null, + query: string, +): Agent[] => { + if (!agents) return []; + if (!query.trim()) return agents; + + const normalizedQuery = query.toLowerCase().trim(); + return agents.filter( + (agent) => + agent.name.toLowerCase().includes(normalizedQuery) || + agent.description?.toLowerCase().includes(normalizedQuery), + ); +}; + +export function useAgentSearch(): UseAgentSearchResult { + const [searchQuery, setSearchQuery] = useState(''); + const [activeFilter, setActiveFilter] = useState('all'); + + const templateAgents = useSelector(selectTemplateAgents); + const userAgents = useSelector(selectAgents); + const sharedAgents = useSelector(selectSharedAgents); + + const handleSearchChange = useCallback((query: string) => { + setSearchQuery(query); + }, []); + + const handleFilterChange = useCallback((filter: AgentFilterTab) => { + setActiveFilter(filter); + }, []); + + const isDataLoaded = useMemo( + (): Record => ({ + template: templateAgents !== null, + user: userAgents !== null, + shared: sharedAgents !== null, + }), + [templateAgents, userAgents, sharedAgents], + ); + + const totalAgentsBySection = useMemo( + (): Record => ({ + template: templateAgents?.length ?? 0, + user: userAgents?.length ?? 0, + shared: sharedAgents?.length ?? 0, + }), + [templateAgents, userAgents, sharedAgents], + ); + + const filteredAgentsBySection = useMemo((): AgentsBySection => { + const filtered = { + template: filterAgentsByQuery(templateAgents, searchQuery), + user: filterAgentsByQuery(userAgents, searchQuery), + shared: filterAgentsByQuery(sharedAgents, searchQuery), + }; + + if (activeFilter === 'all') { + return filtered; + } + + return { + template: activeFilter === 'template' ? filtered.template : [], + user: activeFilter === 'user' ? filtered.user : [], + shared: activeFilter === 'shared' ? filtered.shared : [], + }; + }, [templateAgents, userAgents, sharedAgents, searchQuery, activeFilter]); + + const hasAnyAgents = useMemo(() => { + return ( + totalAgentsBySection.template > 0 || + totalAgentsBySection.user > 0 || + totalAgentsBySection.shared > 0 + ); + }, [totalAgentsBySection]); + + const hasFilteredResults = useMemo(() => { + return ( + filteredAgentsBySection.template.length > 0 || + filteredAgentsBySection.user.length > 0 || + filteredAgentsBySection.shared.length > 0 + ); + }, [filteredAgentsBySection]); + + return { + searchQuery, + setSearchQuery: handleSearchChange, + activeFilter, + setActiveFilter: handleFilterChange, + filteredAgentsBySection, + totalAgentsBySection, + hasAnyAgents, + hasFilteredResults, + isDataLoaded, + }; +} diff --git a/frontend/src/agents/hooks/useAgentsFetch.ts b/frontend/src/agents/hooks/useAgentsFetch.ts new file mode 100644 index 00000000..8f5367d6 --- /dev/null +++ b/frontend/src/agents/hooks/useAgentsFetch.ts @@ -0,0 +1,83 @@ +import { useCallback, useEffect, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; + +import userService from '../../api/services/userService'; +import { + selectToken, + setAgents, + setSharedAgents, + setTemplateAgents, +} from '../../preferences/preferenceSlice'; +import { AgentSectionId } from '../agents.config'; + +interface UseAgentsFetchResult { + isLoading: Record; + isAllLoaded: boolean; +} + +export function useAgentsFetch(): UseAgentsFetchResult { + const dispatch = useDispatch(); + const token = useSelector(selectToken); + + const [isLoading, setIsLoading] = useState>({ + template: true, + user: true, + shared: true, + }); + + const fetchTemplateAgents = useCallback(async () => { + try { + const response = await userService.getTemplateAgents(token); + if (!response.ok) throw new Error('Failed to fetch template agents'); + const data = await response.json(); + dispatch(setTemplateAgents(data)); + } catch (error) { + dispatch(setTemplateAgents([])); + } finally { + setIsLoading((prev) => ({ ...prev, template: false })); + } + }, [token, dispatch]); + + const fetchUserAgents = useCallback(async () => { + try { + const response = await userService.getAgents(token); + if (!response.ok) throw new Error('Failed to fetch user agents'); + const data = await response.json(); + dispatch(setAgents(data)); + } catch (error) { + dispatch(setAgents([])); + } finally { + setIsLoading((prev) => ({ ...prev, user: false })); + } + }, [token, dispatch]); + + const fetchSharedAgents = useCallback(async () => { + try { + const response = await userService.getSharedAgents(token); + if (!response.ok) throw new Error('Failed to fetch shared agents'); + const data = await response.json(); + dispatch(setSharedAgents(data)); + } catch (error) { + dispatch(setSharedAgents([])); + } finally { + setIsLoading((prev) => ({ ...prev, shared: false })); + } + }, [token, dispatch]); + + useEffect(() => { + setIsLoading({ template: true, user: true, shared: true }); + Promise.all([ + fetchTemplateAgents(), + fetchUserAgents(), + fetchSharedAgents(), + ]); + }, [fetchTemplateAgents, fetchUserAgents, fetchSharedAgents]); + + const isAllLoaded = + !isLoading.template && !isLoading.user && !isLoading.shared; + + return { + isLoading, + isAllLoaded, + }; +} diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json index a4b10003..c4b58a3e 100644 --- a/frontend/src/locale/de.json +++ b/frontend/src/locale/de.json @@ -491,6 +491,15 @@ "description": "Entdecke und erstelle benutzerdefinierte Versionen von DocsGPT, die Anweisungen, zusätzliches Wissen und beliebige Kombinationen von Fähigkeiten kombinieren", "newAgent": "Neuer Agent", "backToAll": "Zurück zu allen Agenten", + "searchPlaceholder": "Suchen...", + "noSearchResults": "Keine Agenten gefunden", + "tryDifferentSearch": "Versuche einen anderen Suchbegriff", + "filters": { + "all": "Alle", + "byDocsGPT": "Von DocsGPT", + "byMe": "Von mir", + "shared": "Mit mir geteilt" + }, "sections": { "template": { "title": "Von DocsGPT", diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 69bde744..dbb8bdbe 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -491,6 +491,15 @@ "description": "Discover and create custom versions of DocsGPT that combine instructions, extra knowledge, and any combination of skills", "newAgent": "New Agent", "backToAll": "Back to all agents", + "searchPlaceholder": "Search...", + "noSearchResults": "No agents found", + "tryDifferentSearch": "Try a different search term", + "filters": { + "all": "All", + "byDocsGPT": "By DocsGPT", + "byMe": "By Me", + "shared": "Shared With Me" + }, "sections": { "template": { "title": "By DocsGPT", diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index 65eea4c9..49f09344 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -491,6 +491,15 @@ "description": "Descubre y crea versiones personalizadas de DocsGPT que combinan instrucciones, conocimiento adicional y cualquier combinación de habilidades", "newAgent": "Nuevo Agente", "backToAll": "Volver a todos los agentes", + "searchPlaceholder": "Buscar...", + "noSearchResults": "No se encontraron agentes", + "tryDifferentSearch": "Prueba con un término de búsqueda diferente", + "filters": { + "all": "Todos", + "byDocsGPT": "Por DocsGPT", + "byMe": "Por mí", + "shared": "Compartidos conmigo" + }, "sections": { "template": { "title": "Por DocsGPT", diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index 6be92551..b8458d2c 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -491,6 +491,15 @@ "description": "指示、追加知識、スキルの組み合わせを含むDocsGPTのカスタムバージョンを発見して作成します", "newAgent": "新しいエージェント", "backToAll": "すべてのエージェントに戻る", + "searchPlaceholder": "検索...", + "noSearchResults": "エージェントが見つかりません", + "tryDifferentSearch": "別の検索語をお試しください", + "filters": { + "all": "すべて", + "byDocsGPT": "DocsGPT提供", + "byMe": "自分の", + "shared": "共有された" + }, "sections": { "template": { "title": "DocsGPT提供", diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index fe136ee7..103f3627 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -491,6 +491,15 @@ "description": "Откройте и создайте пользовательские версии DocsGPT, которые объединяют инструкции, дополнительные знания и любую комбинацию навыков", "newAgent": "Новый Агент", "backToAll": "Вернуться ко всем агентам", + "searchPlaceholder": "Поиск...", + "noSearchResults": "Агенты не найдены", + "tryDifferentSearch": "Попробуйте другой поисковый запрос", + "filters": { + "all": "Все", + "byDocsGPT": "От DocsGPT", + "byMe": "Мои", + "shared": "Поделились со мной" + }, "sections": { "template": { "title": "От DocsGPT", diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index 24c8d273..47c0b236 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -491,6 +491,15 @@ "description": "探索並創建結合指令、額外知識和任意技能組合的DocsGPT自訂版本", "newAgent": "新建代理", "backToAll": "返回所有代理", + "searchPlaceholder": "搜尋...", + "noSearchResults": "未找到代理", + "tryDifferentSearch": "請嘗試不同的搜尋詞", + "filters": { + "all": "全部", + "byDocsGPT": "由DocsGPT提供", + "byMe": "我的", + "shared": "與我共享" + }, "sections": { "template": { "title": "由DocsGPT提供", diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index c14c4e74..12b867fa 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -491,6 +491,15 @@ "description": "发现并创建结合指令、额外知识和任意技能组合的DocsGPT自定义版本", "newAgent": "新建代理", "backToAll": "返回所有代理", + "searchPlaceholder": "搜索...", + "noSearchResults": "未找到代理", + "tryDifferentSearch": "请尝试不同的搜索词", + "filters": { + "all": "全部", + "byDocsGPT": "由DocsGPT提供", + "byMe": "我的", + "shared": "与我共享" + }, "sections": { "template": { "title": "由DocsGPT提供", From e68da34c1317185a14928a34fa96956b7cbb48bd Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 4 Dec 2025 15:52:45 +0000 Subject: [PATCH 10/93] feat: implement internal API authentication mechanism --- application/.env_sample | 1 + application/api/internal/routes.py | 12 +++++++++++- application/core/settings.py | 1 + application/worker.py | 9 ++++++++- deployment/docker-compose.yaml | 2 ++ 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/application/.env_sample b/application/.env_sample index 8ab24d2a..c08b2c1d 100644 --- a/application/.env_sample +++ b/application/.env_sample @@ -1,6 +1,7 @@ API_KEY=your_api_key EMBEDDINGS_KEY=your_api_key API_URL=http://localhost:7091 +INTERNAL_KEY=your_internal_key FLASK_APP=application/app.py FLASK_DEBUG=true diff --git a/application/api/internal/routes.py b/application/api/internal/routes.py index f0c2950e..f812a37f 100755 --- a/application/api/internal/routes.py +++ b/application/api/internal/routes.py @@ -1,7 +1,7 @@ import os import datetime import json -from flask import Blueprint, request, send_from_directory +from flask import Blueprint, request, send_from_directory, jsonify from werkzeug.utils import secure_filename from bson.objectid import ObjectId import logging @@ -24,6 +24,16 @@ current_dir = os.path.dirname( internal = Blueprint("internal", __name__) +@internal.before_request +def verify_internal_key(): + """Verify INTERNAL_KEY for all internal endpoint requests.""" + if settings.INTERNAL_KEY: + internal_key = request.headers.get("X-Internal-Key") + if not internal_key or internal_key != settings.INTERNAL_KEY: + logger.warning(f"Unauthorized internal API access attempt from {request.remote_addr}") + return jsonify({"error": "Unauthorized", "message": "Invalid or missing internal key"}), 401 + + @internal.route("/api/download", methods=["get"]) def download_file(): user = secure_filename(request.args.get("user")) diff --git a/application/core/settings.py b/application/core/settings.py index cecbb333..12759d7f 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -62,6 +62,7 @@ class Settings(BaseSettings): CACHE_REDIS_URL: str = "redis://localhost:6379/2" API_URL: str = "http://localhost:7091" # backend url for celery worker + INTERNAL_KEY: Optional[str] = None # internal api key for worker-to-backend auth API_KEY: Optional[str] = None # LLM api key (used by LLM_PROVIDER) diff --git a/application/worker.py b/application/worker.py index 5c5dc367..cbc0dfdc 100755 --- a/application/worker.py +++ b/application/worker.py @@ -109,6 +109,10 @@ def download_file(url, params, dest_path): def upload_index(full_path, file_data): files = None try: + headers = {} + if settings.INTERNAL_KEY: + headers["X-Internal-Key"] = settings.INTERNAL_KEY + if settings.VECTOR_STORE == "faiss": faiss_path = full_path + "/index.faiss" pkl_path = full_path + "/index.pkl" @@ -129,10 +133,13 @@ def upload_index(full_path, file_data): urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data, + headers=headers, ) else: response = requests.post( - urljoin(settings.API_URL, "/api/upload_index"), data=file_data + urljoin(settings.API_URL, "/api/upload_index"), + data=file_data, + headers=headers, ) response.raise_for_status() except (requests.RequestException, FileNotFoundError) as e: diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index 2eef387b..7ca110aa 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -26,6 +26,7 @@ services: - MONGO_URI=mongodb://mongo:27017/docsgpt - CACHE_REDIS_URL=redis://redis:6379/2 - OPENAI_BASE_URL=$OPENAI_BASE_URL + - INTERNAL_KEY=$INTERNAL_KEY ports: - "7091:7091" volumes: @@ -50,6 +51,7 @@ services: - MONGO_URI=mongodb://mongo:27017/docsgpt - API_URL=http://backend:7091 - CACHE_REDIS_URL=redis://redis:6379/2 + - INTERNAL_KEY=$INTERNAL_KEY volumes: - ../application/indexes:/app/indexes - ../application/inputs:/app/inputs From 9a937d2686b737d1628f936c66d03f752683ba5a Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 5 Dec 2025 18:57:39 +0000 Subject: [PATCH 11/93] Feat/small optimisation (#2182) * optimised ram use + celery * Remove VITE_EMBEDDINGS_NAME * fix: timeout on remote embeds --- .env-template | 6 + application/.env_sample | 12 -- application/celery_init.py | 1 + application/celeryconfig.py | 3 + application/core/settings.py | 15 ++- application/vectorstore/base.py | 112 ++++++++++++---- application/vectorstore/embeddings_local.py | 48 +++++++ deployment/docker-compose.yaml | 10 +- frontend/Dockerfile | 2 +- frontend/src/agents/NewAgent.tsx | 33 ++--- frontend/src/components/SourcesPopup.tsx | 125 ++++++++---------- .../src/modals/ShareConversationModal.tsx | 19 +-- 12 files changed, 243 insertions(+), 143 deletions(-) delete mode 100644 application/.env_sample create mode 100644 application/vectorstore/embeddings_local.py diff --git a/.env-template b/.env-template index e93f0363..8b53112c 100644 --- a/.env-template +++ b/.env-template @@ -1,6 +1,12 @@ API_KEY= LLM_NAME=docsgpt VITE_API_STREAMING=true +INTERNAL_KEY= + +# Remote Embeddings (Optional - for using a remote embeddings API instead of local SentenceTransformer) +# When set, the app will use the remote API and won't load SentenceTransformer (saves RAM) +EMBEDDINGS_BASE_URL= +EMBEDDINGS_KEY= #For Azure (you can delete it if you don't use Azure) OPENAI_API_BASE= diff --git a/application/.env_sample b/application/.env_sample deleted file mode 100644 index c08b2c1d..00000000 --- a/application/.env_sample +++ /dev/null @@ -1,12 +0,0 @@ -API_KEY=your_api_key -EMBEDDINGS_KEY=your_api_key -API_URL=http://localhost:7091 -INTERNAL_KEY=your_internal_key -FLASK_APP=application/app.py -FLASK_DEBUG=true - -#For OPENAI on Azure -OPENAI_API_BASE= -OPENAI_API_VERSION= -AZURE_DEPLOYMENT_NAME= -AZURE_EMBEDDINGS_DEPLOYMENT_NAME= \ No newline at end of file diff --git a/application/celery_init.py b/application/celery_init.py index 185cc87f..3e9c3c57 100644 --- a/application/celery_init.py +++ b/application/celery_init.py @@ -21,3 +21,4 @@ def config_loggers(*args, **kwargs): celery = make_celery() +celery.config_from_object("application.celeryconfig") diff --git a/application/celeryconfig.py b/application/celeryconfig.py index 712b3bfc..5a33ee19 100644 --- a/application/celeryconfig.py +++ b/application/celeryconfig.py @@ -6,3 +6,6 @@ result_backend = os.getenv("CELERY_RESULT_BACKEND") task_serializer = 'json' result_serializer = 'json' accept_content = ['json'] + +# Autodiscover tasks +imports = ('application.api.user.tasks',) diff --git a/application/core/settings.py b/application/core/settings.py index 12759d7f..f688df9b 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -2,7 +2,7 @@ import os from pathlib import Path from typing import Optional -from pydantic_settings import BaseSettings +from pydantic_settings import BaseSettings, SettingsConfigDict current_dir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -10,12 +10,19 @@ current_dir = os.path.dirname( class Settings(BaseSettings): + model_config = SettingsConfigDict(extra="ignore") + AUTH_TYPE: Optional[str] = None # simple_jwt, session_jwt, or None LLM_PROVIDER: str = "docsgpt" LLM_NAME: Optional[str] = ( None # if LLM_PROVIDER is openai, LLM_NAME can be gpt-4 or gpt-3.5-turbo ) EMBEDDINGS_NAME: str = "huggingface_sentence-transformers/all-mpnet-base-v2" + EMBEDDINGS_BASE_URL: Optional[str] = None # Remote embeddings API URL (OpenAI-compatible) + EMBEDDINGS_KEY: Optional[str] = ( + None # api key for embeddings (if using openai, just copy API_KEY) + ) + CELERY_BROKER_URL: str = "redis://localhost:6379/0" CELERY_RESULT_BACKEND: str = "redis://localhost:6379/1" MONGO_URI: str = "mongodb://localhost:27017/docsgpt" @@ -73,9 +80,6 @@ class Settings(BaseSettings): GROQ_API_KEY: Optional[str] = None HUGGINGFACE_API_KEY: Optional[str] = None - EMBEDDINGS_KEY: Optional[str] = ( - None # api key for embeddings (if using openai, just copy API_KEY) - ) OPENAI_API_BASE: Optional[str] = None # azure openai api base url OPENAI_API_VERSION: Optional[str] = None # azure openai api version AZURE_DEPLOYMENT_NAME: Optional[str] = None # azure deployment name for answering @@ -153,5 +157,6 @@ class Settings(BaseSettings): COMPRESSION_MAX_HISTORY_POINTS: int = 3 # Keep only last N compression points to prevent DB bloat -path = Path(__file__).parent.parent.absolute() +# Project root is one level above application/ +path = Path(__file__).parent.parent.parent.absolute() settings = Settings(_env_file=path.joinpath(".env"), _env_file_encoding="utf-8") diff --git a/application/vectorstore/base.py b/application/vectorstore/base.py index 84839059..e5c65794 100644 --- a/application/vectorstore/base.py +++ b/application/vectorstore/base.py @@ -2,41 +2,79 @@ import logging import os from abc import ABC, abstractmethod +import requests from langchain_openai import OpenAIEmbeddings -from sentence_transformers import SentenceTransformer from application.core.settings import settings -class EmbeddingsWrapper: - def __init__(self, model_name, *args, **kwargs): - logging.info(f"Initializing EmbeddingsWrapper with model: {model_name}") - try: - kwargs.setdefault("trust_remote_code", True) - self.model = SentenceTransformer( - model_name, - config_kwargs={"allow_dangerous_deserialization": True}, - *args, - **kwargs, - ) - if self.model is None or self.model._first_module() is None: +class RemoteEmbeddings: + """ + Wrapper for remote embeddings API (OpenAI-compatible). + Used when EMBEDDINGS_BASE_URL is configured. + """ + + def __init__(self, api_url: str, model_name: str, api_key: str = None): + self.api_url = api_url.rstrip("/") + self.model_name = model_name + self.headers = {"Content-Type": "application/json"} + if api_key: + self.headers["Authorization"] = f"Bearer {api_key}" + self.dimension = None + + def _embed(self, inputs): + """Send embedding request to remote API.""" + payload = {"inputs": inputs} + if self.model_name: + payload["model"] = self.model_name + + response = requests.post( + self.api_url, headers=self.headers, json=payload, timeout=180 + ) + response.raise_for_status() + result = response.json() + + if isinstance(result, list): + if result and isinstance(result[0], list): + return result + elif result and all(isinstance(x, (int, float)) for x in result): + return [result] + elif not result: + return [] + else: raise ValueError( - f"SentenceTransformer model failed to load properly for: {model_name}" + f"Unexpected list content from remote embeddings API: {result}" ) - self.dimension = self.model.get_sentence_embedding_dimension() - logging.info(f"Successfully loaded model with dimension: {self.dimension}") - except Exception as e: - logging.error( - f"Failed to initialize SentenceTransformer with model {model_name}: {str(e)}", - exc_info=True, + elif isinstance(result, dict) and "error" in result: + raise ValueError(f"Remote embeddings API error: {result['error']}") + else: + raise ValueError( + f"Unexpected response format from remote embeddings API: {result}" ) - raise def embed_query(self, query: str): - return self.model.encode(query).tolist() + """Embed a single query string.""" + embeddings_list = self._embed(query) + if ( + isinstance(embeddings_list, list) + and len(embeddings_list) == 1 + and isinstance(embeddings_list[0], list) + ): + if self.dimension is None: + self.dimension = len(embeddings_list[0]) + return embeddings_list[0] + raise ValueError( + f"Unexpected result structure after embedding query: {embeddings_list}" + ) def embed_documents(self, documents: list): - return self.model.encode(documents).tolist() + """Embed a list of documents.""" + if not documents: + return [] + embeddings_list = self._embed(documents) + if self.dimension is None and embeddings_list: + self.dimension = len(embeddings_list[0]) + return embeddings_list def __call__(self, text): if isinstance(text, str): @@ -47,6 +85,13 @@ class EmbeddingsWrapper: raise ValueError("Input must be a string or a list of strings") +def _get_embeddings_wrapper(): + """Lazy import of EmbeddingsWrapper to avoid loading SentenceTransformer when using remote embeddings.""" + from application.vectorstore.embeddings_local import EmbeddingsWrapper + + return EmbeddingsWrapper + + class EmbeddingsSingleton: _instances = {} @@ -60,8 +105,13 @@ class EmbeddingsSingleton: @staticmethod def _create_instance(embeddings_name, *args, **kwargs): + if embeddings_name == "openai_text-embedding-ada-002": + return OpenAIEmbeddings(*args, **kwargs) + + # Lazy import EmbeddingsWrapper only when needed (avoids loading SentenceTransformer) + EmbeddingsWrapper = _get_embeddings_wrapper() + embeddings_factory = { - "openai_text-embedding-ada-002": OpenAIEmbeddings, "huggingface_sentence-transformers/all-mpnet-base-v2": lambda: EmbeddingsWrapper( "sentence-transformers/all-mpnet-base-v2" ), @@ -121,6 +171,20 @@ class BaseVectorStore(ABC): ) def _get_embeddings(self, embeddings_name, embeddings_key=None): + # Check for remote embeddings first + if settings.EMBEDDINGS_BASE_URL: + logging.info( + f"Using remote embeddings API at: {settings.EMBEDDINGS_BASE_URL}" + ) + cache_key = f"remote_{settings.EMBEDDINGS_BASE_URL}_{embeddings_name}" + if cache_key not in EmbeddingsSingleton._instances: + EmbeddingsSingleton._instances[cache_key] = RemoteEmbeddings( + api_url=settings.EMBEDDINGS_BASE_URL, + model_name=embeddings_name, + api_key=embeddings_key, + ) + return EmbeddingsSingleton._instances[cache_key] + if embeddings_name == "openai_text-embedding-ada-002": if self.is_azure_configured(): os.environ["OPENAI_API_TYPE"] = "azure" diff --git a/application/vectorstore/embeddings_local.py b/application/vectorstore/embeddings_local.py new file mode 100644 index 00000000..fb6e5f4f --- /dev/null +++ b/application/vectorstore/embeddings_local.py @@ -0,0 +1,48 @@ +""" +Local embeddings using SentenceTransformer. +This module is only imported when EMBEDDINGS_BASE_URL is not set, +to avoid loading SentenceTransformer into memory when using remote embeddings. +""" + +import logging + +from sentence_transformers import SentenceTransformer + + +class EmbeddingsWrapper: + def __init__(self, model_name, *args, **kwargs): + logging.info(f"Initializing EmbeddingsWrapper with model: {model_name}") + try: + kwargs.setdefault("trust_remote_code", True) + self.model = SentenceTransformer( + model_name, + config_kwargs={"allow_dangerous_deserialization": True}, + *args, + **kwargs, + ) + if self.model is None or self.model._first_module() is None: + raise ValueError( + f"SentenceTransformer model failed to load properly for: {model_name}" + ) + self.dimension = self.model.get_sentence_embedding_dimension() + logging.info(f"Successfully loaded model with dimension: {self.dimension}") + except Exception as e: + logging.error( + f"Failed to initialize SentenceTransformer with model {model_name}: {str(e)}", + exc_info=True, + ) + raise + + def embed_query(self, query: str): + return self.model.encode(query).tolist() + + def embed_documents(self, documents: list): + return self.model.encode(documents).tolist() + + def __call__(self, text): + if isinstance(text, str): + return self.embed_query(text) + elif isinstance(text, list): + return self.embed_documents(text) + else: + raise ValueError("Input must be a string or a list of strings") diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index 7ca110aa..91c22409 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -16,9 +16,12 @@ services: backend: user: root build: ../application + env_file: + - ../.env environment: - API_KEY=$API_KEY - - EMBEDDINGS_KEY=$API_KEY + - EMBEDDINGS_KEY=$EMBEDDINGS_KEY + - EMBEDDINGS_BASE_URL=$EMBEDDINGS_BASE_URL - LLM_PROVIDER=$LLM_PROVIDER - LLM_NAME=$LLM_NAME - CELERY_BROKER_URL=redis://redis:6379/0 @@ -41,9 +44,12 @@ services: user: root build: ../application command: celery -A application.app.celery worker -l INFO -B + env_file: + - ../.env environment: - API_KEY=$API_KEY - - EMBEDDINGS_KEY=$API_KEY + - EMBEDDINGS_KEY=$EMBEDDINGS_KEY + - EMBEDDINGS_BASE_URL=$EMBEDDINGS_BASE_URL - LLM_PROVIDER=$LLM_PROVIDER - LLM_NAME=$LLM_NAME - CELERY_BROKER_URL=redis://redis:6379/0 diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 5ca21455..19574bf5 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.6.1-bullseye-slim +FROM node:22-bullseye-slim WORKDIR /app diff --git a/frontend/src/agents/NewAgent.tsx b/frontend/src/agents/NewAgent.tsx index 1c6b194b..c9605a26 100644 --- a/frontend/src/agents/NewAgent.tsx +++ b/frontend/src/agents/NewAgent.tsx @@ -28,9 +28,6 @@ import AgentPreview from './AgentPreview'; import { Agent, ToolSummary } from './types'; import type { Model } from '../models/types'; -const embeddingsName = - import.meta.env.VITE_EMBEDDINGS_NAME || - 'huggingface_sentence-transformers/all-mpnet-base-v2'; export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { const { t } = useTranslation(); @@ -548,22 +545,20 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { } else { // Single source selected - maintain backward compatibility const selectedSource = selectedSources[0]; - if (selectedSource?.model === embeddingsName) { - if (selectedSource && 'id' in selectedSource) { - setAgent((prev) => ({ - ...prev, - source: selectedSource?.id || 'default', - sources: [], // Clear sources array for single source - retriever: '', - })); - } else { - setAgent((prev) => ({ - ...prev, - source: '', - sources: [], // Clear sources array - retriever: selectedSource?.retriever || 'classic', - })); - } + if (selectedSource && 'id' in selectedSource) { + setAgent((prev) => ({ + ...prev, + source: selectedSource?.id || 'default', + sources: [], // Clear sources array for single source + retriever: '', + })); + } else { + setAgent((prev) => ({ + ...prev, + source: '', + sources: [], // Clear sources array + retriever: selectedSource?.retriever || 'classic', + })); } } } else { diff --git a/frontend/src/components/SourcesPopup.tsx b/frontend/src/components/SourcesPopup.tsx index c1b4d045..8ae399ba 100644 --- a/frontend/src/components/SourcesPopup.tsx +++ b/frontend/src/components/SourcesPopup.tsx @@ -40,10 +40,6 @@ export default function SourcesPopup({ showAbove: false, }); - const embeddingsName = - import.meta.env.VITE_EMBEDDINGS_NAME || - 'huggingface_sentence-transformers/all-mpnet-base-v2'; - const options = useSelector(selectSourceDocs); const selectedDocs = useSelector(selectSelectedDocs); @@ -147,70 +143,65 @@ export default function SourcesPopup({ {options ? ( <> {filteredOptions?.map((option: any, index: number) => { - if (option.model === embeddingsName) { - const isSelected = - selectedDocs && - Array.isArray(selectedDocs) && - selectedDocs.length > 0 && - selectedDocs.some((doc) => - option.id - ? doc.id === option.id - : doc.date === option.date, - ); - - return ( -
{ - if (isSelected) { - const updatedDocs = - selectedDocs && Array.isArray(selectedDocs) - ? selectedDocs.filter((doc) => - option.id - ? doc.id !== option.id - : doc.date !== option.date, - ) - : []; - dispatch(setSelectedDocs(updatedDocs)); - handlePostDocumentSelect( - updatedDocs.length > 0 ? updatedDocs : null, - ); - } else { - const updatedDocs = - selectedDocs && Array.isArray(selectedDocs) - ? [...selectedDocs, option] - : [option]; - dispatch(setSelectedDocs(updatedDocs)); - handlePostDocumentSelect(updatedDocs); - } - }} - > - Source - - {option.name} - -
- {isSelected && ( - Selected - )} -
-
+ const isSelected = + selectedDocs && + Array.isArray(selectedDocs) && + selectedDocs.length > 0 && + selectedDocs.some((doc) => + option.id ? doc.id === option.id : doc.date === option.date, ); - } - return null; + + return ( +
{ + if (isSelected) { + const updatedDocs = + selectedDocs && Array.isArray(selectedDocs) + ? selectedDocs.filter((doc) => + option.id + ? doc.id !== option.id + : doc.date !== option.date, + ) + : []; + dispatch(setSelectedDocs(updatedDocs)); + handlePostDocumentSelect( + updatedDocs.length > 0 ? updatedDocs : null, + ); + } else { + const updatedDocs = + selectedDocs && Array.isArray(selectedDocs) + ? [...selectedDocs, option] + : [option]; + dispatch(setSelectedDocs(updatedDocs)); + handlePostDocumentSelect(updatedDocs); + } + }} + > + Source + + {option.name} + +
+ {isSelected && ( + Selected + )} +
+
+ ); })} ) : ( diff --git a/frontend/src/modals/ShareConversationModal.tsx b/frontend/src/modals/ShareConversationModal.tsx index 1dddef6f..921fd59f 100644 --- a/frontend/src/modals/ShareConversationModal.tsx +++ b/frontend/src/modals/ShareConversationModal.tsx @@ -16,11 +16,6 @@ import { } from '../preferences/preferenceSlice'; import WrapperModal from './WrapperModal'; -const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; -const embeddingsName = - import.meta.env.VITE_EMBEDDINGS_NAME || - 'huggingface_sentence-transformers/all-mpnet-base-v2'; - type StatusType = 'loading' | 'idle' | 'fetched' | 'failed'; export const ShareConversationModal = ({ @@ -47,14 +42,12 @@ export const ShareConversationModal = ({ const extractDocPaths = (docs: Doc[]) => docs - ? docs - .filter((doc) => doc.model === embeddingsName) - .map((doc: Doc) => { - return { - label: doc.name, - value: doc.id ?? 'default', - }; - }) + ? docs.map((doc: Doc) => { + return { + label: doc.name, + value: doc.id ?? 'default', + }; + }) : []; const [sourcePath, setSourcePath] = useState<{ From 4adffe762aba2c346776849b135ec32a77ec958c Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 8 Dec 2025 16:59:08 +0200 Subject: [PATCH 12/93] Update README.md --- README.md | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 46016bc2..92a39e31 100644 --- a/README.md +++ b/README.md @@ -46,24 +46,10 @@ ## Roadmap - -- [x] Full GoogleAI compatibility (Jan 2025) -- [x] Add tools (Jan 2025) -- [x] Manually updating chunks in the app UI (Feb 2025) -- [x] Devcontainer for easy development (Feb 2025) -- [x] ReACT agent (March 2025) -- [x] Chatbots menu re-design to handle tools, agent types, and more (April 2025) -- [x] New input box in the conversation menu (April 2025) -- [x] Add triggerable actions / tools (webhook) (April 2025) -- [x] Agent optimisations (May 2025) -- [x] Filesystem sources update (July 2025) -- [x] Json Responses (August 2025) -- [x] MCP support (August 2025) -- [x] Google Drive integration (September 2025) - [x] Add OAuth 2.0 authentication for MCP (September 2025) -- [ ] SharePoint integration (October 2025) -- [ ] Deep Agents (October 2025) -- [ ] Agent scheduling +- [x] Deep Agents (October 2025) +- [ ] Prompt Templating ( October 2025 ) +- [ ] Agent scheduling ( December 2025 ) You can find our full roadmap [here](https://github.com/orgs/arc53/projects/2). Please don't hesitate to contribute or create issues, it helps us improve DocsGPT! @@ -158,9 +144,17 @@ We as members, contributors, and leaders, pledge to make participation in our co The source code license is [MIT](https://opensource.org/license/mit/), as described in the [LICENSE](LICENSE) file. -

This project is supported by:

+## This project is supported by: +

+

+ + color + +

+ + From 09e7c1b97fbdb953458f6a14cf24d7ff19c01386 Mon Sep 17 00:00:00 2001 From: Manish Madan Date: Thu, 11 Dec 2025 03:23:40 +0530 Subject: [PATCH 13/93] Fixes: re-blink in converstaion, (refactor) prompts and validate LocalStorage prompts (#2181) * chore(dependabot): add react-widget npm dependency updates * refactor(prompts): init on load, mv to pref slice * (refactor): searchable dropdowns are separate * (fix/ui) prompts adjust * feat(changelog): dancing stars * (fix)conversation: re-blink bubble past stream * (fix)endless GET sources, esling err --------- Co-authored-by: GH Action - Upstream Sync --- frontend/src/App.tsx | 2 + frontend/src/Navigation.tsx | 3 - frontend/src/agents/NewAgent.tsx | 36 +-- frontend/src/assets/search.svg | 4 +- frontend/src/components/Notification.tsx | 140 +++++++-- .../src/components/SearchableDropdown.tsx | 271 ++++++++++++++++++ .../src/conversation/ConversationMessages.tsx | 11 +- frontend/src/hooks/useDataInitializer.ts | 111 +++++++ frontend/src/hooks/useDefaultDocument.ts | 37 --- frontend/src/preferences/preferenceApi.ts | 49 +++- frontend/src/preferences/preferenceSlice.ts | 35 ++- frontend/src/settings/General.tsx | 27 +- frontend/src/settings/Prompts.tsx | 11 +- frontend/src/store.ts | 5 + 14 files changed, 613 insertions(+), 129 deletions(-) create mode 100644 frontend/src/components/SearchableDropdown.tsx create mode 100644 frontend/src/hooks/useDataInitializer.ts delete mode 100644 frontend/src/hooks/useDefaultDocument.ts diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 0c3384d1..ec50b83d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -11,6 +11,7 @@ import UploadToast from './components/UploadToast'; import Conversation from './conversation/Conversation'; import { SharedConversation } from './conversation/SharedConversation'; import { useDarkTheme, useMediaQuery } from './hooks'; +import useDataInitializer from './hooks/useDataInitializer'; import useTokenAuth from './hooks/useTokenAuth'; import Navigation from './Navigation'; import PageNotFound from './PageNotFound'; @@ -19,6 +20,7 @@ import Notification from './components/Notification'; function AuthWrapper({ children }: { children: React.ReactNode }) { const { isAuthLoading } = useTokenAuth(); + useDataInitializer(isAuthLoading); if (isAuthLoading) { return ( diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index 1847d8f9..b556033d 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -31,7 +31,6 @@ import { } from './conversation/conversationSlice'; import ConversationTile from './conversation/ConversationTile'; import { useDarkTheme, useMediaQuery } from './hooks'; -import useDefaultDocument from './hooks/useDefaultDocument'; import useTokenAuth from './hooks/useTokenAuth'; import DeleteConvModal from './modals/DeleteConvModal'; import JWTModal from './modals/JWTModal'; @@ -155,7 +154,6 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { }, [agents, sharedAgents, token, dispatch]); useEffect(() => { - if (!conversations?.data) fetchConversations(); if (queries.length === 0) resetConversation(); }, [conversations?.data, dispatch]); @@ -290,7 +288,6 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { setNavOpen(!(isMobile || isTablet)); }, [isMobile, isTablet]); - useDefaultDocument(); return ( <> {(isMobile || isTablet) && navOpen && ( diff --git a/frontend/src/agents/NewAgent.tsx b/frontend/src/agents/NewAgent.tsx index c9605a26..d22476af 100644 --- a/frontend/src/agents/NewAgent.tsx +++ b/frontend/src/agents/NewAgent.tsx @@ -19,7 +19,9 @@ import { selectSelectedAgent, selectSourceDocs, selectToken, + selectPrompts, setSelectedAgent, + setPrompts, } from '../preferences/preferenceSlice'; import PromptsModal from '../preferences/PromptsModal'; import Prompts from '../settings/Prompts'; @@ -38,6 +40,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { const token = useSelector(selectToken); const sourceDocs = useSelector(selectSourceDocs); const selectedAgent = useSelector(selectSelectedAgent); + const prompts = useSelector(selectPrompts); const [effectiveMode, setEffectiveMode] = useState(mode); const [agent, setAgent] = useState({ @@ -62,9 +65,6 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { default_model_id: '', }); const [imageFile, setImageFile] = useState(null); - const [prompts, setPrompts] = useState< - { name: string; id: string; type: string }[] - >([]); const [userTools, setUserTools] = useState([]); const [availableModels, setAvailableModels] = useState([]); const [isSourcePopupOpen, setIsSourcePopupOpen] = useState(false); @@ -401,14 +401,6 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { })); setUserTools(tools); }; - const getPrompts = async () => { - const response = await userService.getPrompts(token); - if (!response.ok) { - throw new Error('Failed to fetch prompts'); - } - const data = await response.json(); - setPrompts(data); - }; const getModels = async () => { const response = await modelService.getModels(null); if (!response.ok) throw new Error('Failed to fetch models'); @@ -417,7 +409,6 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { setAvailableModels(transformed); }; getTools(); - getPrompts(); getModels(); }, [token]); @@ -604,7 +595,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { setHasChanges(isChanged); }, [agent, dispatch, effectiveMode, imageFile, jsonSchemaText]); return ( -
+
- + {/* Animated stars background */} +
+ {stars.map((star) => ( + + {/* 4-pointed Christmas star */} + + + ))} +
+ +

+ {notificationText} +

+ + + {/* Arrow tail - grows leftward from arrow head's back point on hover */} + + {/* Arrow head - pushed forward by the tail on hover */} + + + + + + + ); } diff --git a/frontend/src/components/SearchableDropdown.tsx b/frontend/src/components/SearchableDropdown.tsx new file mode 100644 index 00000000..efd2b8a4 --- /dev/null +++ b/frontend/src/components/SearchableDropdown.tsx @@ -0,0 +1,271 @@ +import React from 'react'; + +import Arrow2 from '../assets/dropdown-arrow.svg'; +import Edit from '../assets/edit.svg'; +import Search from '../assets/search.svg'; +import Trash from '../assets/trash.svg'; + +/** + * SearchableDropdown - A standalone dropdown component with built-in search functionality + */ + +type SearchableDropdownOptionBase = { + id?: string; + type?: string; +}; + +type NameIdOption = { name: string; id: string } & SearchableDropdownOptionBase; + +export type SearchableDropdownOption = + | string + | NameIdOption + | ({ label: string; value: string } & SearchableDropdownOptionBase) + | ({ value: number; description: string } & SearchableDropdownOptionBase); + +export type SearchableDropdownSelectedValue = SearchableDropdownOption | null; + +export interface SearchableDropdownProps< + T extends SearchableDropdownOption = SearchableDropdownOption, +> { + options: T[]; + selectedValue: SearchableDropdownSelectedValue; + onSelect: (value: T) => void; + size?: string; + /** Controls border radius for both button and dropdown menu */ + rounded?: 'xl' | '3xl'; + border?: 'border' | 'border-2'; + showEdit?: boolean; + onEdit?: (value: NameIdOption) => void; + showDelete?: boolean | ((option: T) => boolean); + onDelete?: (id: string) => void; + placeholder?: string; +} + +function SearchableDropdown({ + options, + selectedValue, + onSelect, + size = 'w-32', + rounded = 'xl', + border = 'border-2', + showEdit, + onEdit, + showDelete, + onDelete, + placeholder, +}: SearchableDropdownProps) { + const dropdownRef = React.useRef(null); + const searchInputRef = React.useRef(null); + const [isOpen, setIsOpen] = React.useState(false); + const [searchQuery, setSearchQuery] = React.useState(''); + + const borderRadius = rounded === 'xl' ? 'rounded-xl' : 'rounded-3xl'; + + React.useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + dropdownRef.current && + !dropdownRef.current.contains(event.target as Node) + ) { + setIsOpen(false); + setSearchQuery(''); + } + }; + + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + }, []); + + React.useEffect(() => { + if (isOpen && searchInputRef.current) { + searchInputRef.current.focus(); + } + }, [isOpen]); + + const getOptionText = (option: SearchableDropdownOption): string => { + if (typeof option === 'string') return option; + if ('name' in option) return option.name; + if ('label' in option) return option.label; + if ('description' in option) return option.description; + return ''; + }; + + const filteredOptions = React.useMemo(() => { + if (!searchQuery.trim()) return options; + const query = searchQuery.toLowerCase(); + return options.filter((option) => + getOptionText(option).toLowerCase().includes(query), + ); + }, [options, searchQuery]); + + const getDisplayValue = (): string => { + if (!selectedValue) return placeholder ?? 'From URL'; + if (typeof selectedValue === 'string') return selectedValue; + if ('label' in selectedValue) return selectedValue.label; + if ('name' in selectedValue) return selectedValue.name; + if ('description' in selectedValue) { + return selectedValue.value < 1e9 + ? `${selectedValue.value} (${selectedValue.description})` + : selectedValue.description; + } + return placeholder ?? 'From URL'; + }; + + const isOptionSelected = (option: T): boolean => { + if (!selectedValue) return false; + if (typeof selectedValue === 'string') + return selectedValue === (option as unknown as string); + if (typeof option === 'string') return false; + + const optionObj = option as Record; + const selectedObj = selectedValue as Record; + + if ('name' in optionObj && 'name' in selectedObj) + return selectedObj.name === optionObj.name; + if ('label' in optionObj && 'label' in selectedObj) + return selectedObj.label === optionObj.label; + if ('value' in optionObj && 'value' in selectedObj) + return selectedObj.value === optionObj.value; + return false; + }; + + return ( +
+ + + {isOpen && ( +
+
+
+ search + setSearchQuery(e.target.value)} + placeholder="Search..." + className="dark:text-bright-gray w-full rounded-lg border-0 bg-transparent py-2 pr-3 pl-10 font-['Inter'] text-[14px] leading-[16.5px] font-normal focus:ring-0 focus:outline-none" + onClick={(e) => e.stopPropagation()} + /> +
+
+ +
+ {filteredOptions.length === 0 ? ( +
+ No results found +
+ ) : ( + filteredOptions.map((option, index) => { + const selected = isOptionSelected(option); + const optionObj = + typeof option !== 'string' + ? (option as Record) + : null; + const optionType = optionObj?.type as string | undefined; + const optionId = optionObj?.id as string | undefined; + const optionName = optionObj?.name as string | undefined; + + return ( +
+ { + onSelect(option); + setIsOpen(false); + setSearchQuery(''); + }} + className="dark:text-light-gray ml-5 flex-1 overflow-hidden py-3 font-['Inter'] text-[14px] leading-[16.5px] font-normal text-ellipsis whitespace-nowrap" + > + {getOptionText(option)} + + {showEdit && + onEdit && + optionObj && + optionType !== 'public' && ( + Edit { + if (optionName && optionId) { + onEdit({ + id: optionId, + name: optionName, + type: optionType, + }); + } + setIsOpen(false); + setSearchQuery(''); + }} + /> + )} + {showDelete && onDelete && ( + + )} +
+ ); + }) + )} +
+
+ )} +
+ ); +} + +export default SearchableDropdown; diff --git a/frontend/src/conversation/ConversationMessages.tsx b/frontend/src/conversation/ConversationMessages.tsx index 0007f5dc..bd099f12 100644 --- a/frontend/src/conversation/ConversationMessages.tsx +++ b/frontend/src/conversation/ConversationMessages.tsx @@ -7,7 +7,6 @@ import { useState, } from 'react'; import { useTranslation } from 'react-i18next'; -import { useSelector } from 'react-redux'; import ArrowDown from '../assets/arrow-down.svg'; import RetryIcon from '../components/RetryIcon'; @@ -15,7 +14,6 @@ import Hero from '../Hero'; import { useDarkTheme } from '../hooks'; import ConversationBubble from './ConversationBubble'; import { FEEDBACK, Query, Status } from './conversationModels'; -import { selectConversationId } from '../preferences/preferenceSlice'; const SCROLL_THRESHOLD = 10; const LAST_BUBBLE_MARGIN = 'mb-32'; @@ -52,7 +50,6 @@ export default function ConversationMessages({ }: ConversationMessagesProps) { const [isDarkTheme] = useDarkTheme(); const { t } = useTranslation(); - const conversationId = useSelector(selectConversationId); const conversationRef = useRef(null); const [hasScrolledToLast, setHasScrolledToLast] = useState(true); @@ -145,7 +142,7 @@ export default function ConversationMessages({ return ( 0 ? ( queries.map((query, index) => ( - + { + // Skip if auth is still loading + if (isAuthLoading) { + return; + } + + const fetchDocs = async () => { + try { + const data = await getDocs(token); + dispatch(setSourceDocs(data)); + + // Auto-select default document if none selected + if ( + !selectedDoc || + (Array.isArray(selectedDoc) && selectedDoc.length === 0) + ) { + if (Array.isArray(data)) { + data.forEach((doc: Doc) => { + if (doc.model && doc.name === 'default') { + dispatch(setSelectedDocs([doc])); + } + }); + } + } + } catch (error) { + console.error('Failed to fetch documents:', error); + } + }; + + fetchDocs(); + }, [isAuthLoading, token]); + + // Initialize prompts + useEffect(() => { + // Skip if auth is still loading + if (isAuthLoading) { + return; + } + + const fetchPromptsData = async () => { + try { + const data = await getPrompts(token); + dispatch(setPrompts(data)); + } catch (error) { + console.error('Failed to fetch prompts:', error); + } + }; + + fetchPromptsData(); + }, [isAuthLoading, token]); + + // Initialize conversations + useEffect(() => { + // Skip if auth is still loading + if (isAuthLoading) { + return; + } + + const fetchConversationsData = async () => { + if (!conversations?.data) { + dispatch(setConversations({ ...conversations, loading: true })); + try { + const fetchedConversations = await getConversations(token); + dispatch(setConversations(fetchedConversations)); + } catch (error) { + console.error('Failed to fetch conversations:', error); + dispatch(setConversations({ data: null, loading: false })); + } + } + }; + + fetchConversationsData(); + }, [isAuthLoading, conversations?.data, token]); +} diff --git a/frontend/src/hooks/useDefaultDocument.ts b/frontend/src/hooks/useDefaultDocument.ts deleted file mode 100644 index 17568c59..00000000 --- a/frontend/src/hooks/useDefaultDocument.ts +++ /dev/null @@ -1,37 +0,0 @@ -import React from 'react'; -import { useDispatch, useSelector } from 'react-redux'; - -import { Doc } from '../models/misc'; -import { getDocs } from '../preferences/preferenceApi'; -import { - selectSelectedDocs, - selectToken, - setSelectedDocs, - setSourceDocs, -} from '../preferences/preferenceSlice'; - -export default function useDefaultDocument() { - const dispatch = useDispatch(); - const token = useSelector(selectToken); - const selectedDoc = useSelector(selectSelectedDocs); - - const fetchDocs = () => { - getDocs(token).then((data) => { - dispatch(setSourceDocs(data)); - if ( - !selectedDoc || - (Array.isArray(selectedDoc) && selectedDoc.length === 0) - ) - Array.isArray(data) && - data?.forEach((doc: Doc) => { - if (doc.model && doc.name === 'default') { - dispatch(setSelectedDocs([doc])); - } - }); - }); - }; - - React.useEffect(() => { - fetchDocs(); - }, []); -} diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index b9c199e2..39396959 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -1,6 +1,6 @@ import conversationService from '../api/services/conversationService'; import userService from '../api/services/userService'; -import { Doc, GetDocsResponse } from '../models/misc'; +import { Doc, GetDocsResponse, Prompt } from '../models/misc'; import { GetConversationsResult, ConversationSummary } from './types'; //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later. @@ -113,17 +113,40 @@ export function getLocalRecentDocs(sourceDocs?: Doc[] | null): Doc[] | null { return validDocs.length > 0 ? validDocs : null; } -export function getLocalPrompt(): string | null { - const prompt = localStorage.getItem('DocsGPTPrompt'); - return prompt; +export function getLocalPrompt( + availablePrompts?: Prompt[] | null, +): Prompt | null { + const promptString = localStorage.getItem('DocsGPTPrompt'); + const selectedPrompt = promptString + ? (JSON.parse(promptString) as Prompt) + : null; + + if (!availablePrompts || !selectedPrompt) { + return selectedPrompt; + } + + const isPromptAvailable = (selected: Prompt) => { + return availablePrompts.some((available) => { + return available.id === selected.id; + }); + }; + + const isValid = isPromptAvailable(selectedPrompt); + + if (!isValid) { + localStorage.removeItem('DocsGPTPrompt'); + return null; + } + + return selectedPrompt; } export function setLocalApiKey(key: string): void { localStorage.setItem('DocsGPTApiKey', key); } -export function setLocalPrompt(prompt: string): void { - localStorage.setItem('DocsGPTPrompt', prompt); +export function setLocalPrompt(prompt: Prompt): void { + localStorage.setItem('DocsGPTPrompt', JSON.stringify(prompt)); } export function setLocalRecentDocs(docs: Doc[] | null): void { @@ -133,3 +156,17 @@ export function setLocalRecentDocs(docs: Doc[] | null): void { localStorage.removeItem('DocsGPTRecentDocs'); } } + +export async function getPrompts(token: string | null): Promise { + try { + const response = await userService.getPrompts(token); + if (!response.ok) { + throw new Error('Failed to fetch prompts'); + } + const data = await response.json(); + return data as Prompt[]; + } catch (error) { + console.error('Error fetching prompts:', error); + return []; + } +} diff --git a/frontend/src/preferences/preferenceSlice.ts b/frontend/src/preferences/preferenceSlice.ts index b3fa7fcd..2bd5faea 100644 --- a/frontend/src/preferences/preferenceSlice.ts +++ b/frontend/src/preferences/preferenceSlice.ts @@ -6,9 +6,10 @@ import { } from '@reduxjs/toolkit'; import { Agent } from '../agents/types'; -import { ActiveState, Doc } from '../models/misc'; +import { ActiveState, Doc, Prompt } from '../models/misc'; import { RootState } from '../store'; import { + getLocalPrompt, getLocalRecentDocs, setLocalApiKey, setLocalRecentDocs, @@ -18,6 +19,7 @@ import type { Model } from '../models/types'; export interface Preference { apiKey: string; prompt: { name: string; id: string; type: string }; + prompts: Prompt[]; chunks: string; token_limit: number; selectedDocs: Doc[]; @@ -41,6 +43,11 @@ export interface Preference { const initialState: Preference = { apiKey: 'xxx', prompt: { name: 'default', id: 'default', type: 'public' }, + prompts: [ + { name: 'default', id: 'default', type: 'public' }, + { name: 'creative', id: 'creative', type: 'public' }, + { name: 'strict', id: 'strict', type: 'public' }, + ], chunks: '2', token_limit: 2000, selectedDocs: [ @@ -95,6 +102,9 @@ export const prefSlice = createSlice({ setPrompt: (state, action) => { state.prompt = action.payload; }, + setPrompts: (state, action: PayloadAction) => { + state.prompts = action.payload; + }, setChunks: (state, action) => { state.chunks = action.payload; }, @@ -135,6 +145,7 @@ export const { setConversations, setToken, setPrompt, + setPrompts, setChunks, setTokenLimit, setModalStateDeleteConv, @@ -217,6 +228,27 @@ prefListenerMiddleware.startListening({ }, }); +prefListenerMiddleware.startListening({ + matcher: isAnyOf(setPrompts), + effect: (_action, listenerApi) => { + const state = listenerApi.getState() as RootState; + const availablePrompts = state.preference.prompts; + if (availablePrompts && availablePrompts.length > 0) { + const validatedPrompt = getLocalPrompt(availablePrompts); + if (validatedPrompt !== null) { + listenerApi.dispatch(setPrompt(validatedPrompt)); + } else { + const defaultPrompt = + availablePrompts.find((p) => p.id === 'default') || + availablePrompts[0]; + if (defaultPrompt) { + listenerApi.dispatch(setPrompt(defaultPrompt)); + } + } + } + }, +}); + prefListenerMiddleware.startListening({ matcher: isAnyOf(setSelectedModel), effect: (action, listenerApi) => { @@ -247,6 +279,7 @@ export const selectConversationId = (state: RootState) => state.conversation.conversationId; export const selectToken = (state: RootState) => state.preference.token; export const selectPrompt = (state: RootState) => state.preference.prompt; +export const selectPrompts = (state: RootState) => state.preference.prompts; export const selectChunks = (state: RootState) => state.preference.chunks; export const selectTokenLimit = (state: RootState) => state.preference.token_limit; diff --git a/frontend/src/settings/General.tsx b/frontend/src/settings/General.tsx index 442f7076..8b2c53cc 100644 --- a/frontend/src/settings/General.tsx +++ b/frontend/src/settings/General.tsx @@ -2,17 +2,17 @@ import React from 'react'; import { useTranslation } from 'react-i18next'; import { useDispatch, useSelector } from 'react-redux'; -import userService from '../api/services/userService'; import Dropdown from '../components/Dropdown'; import { useDarkTheme } from '../hooks'; import { selectChunks, selectPrompt, - selectToken, + selectPrompts, selectTokenLimit, setChunks, setModalStateDeleteConv, setPrompt, + setPrompts, setTokenLimit, } from '../preferences/preferenceSlice'; import Prompts from './Prompts'; @@ -22,7 +22,6 @@ export default function General() { t, i18n: { changeLanguage }, } = useTranslation(); - const token = useSelector(selectToken); const themes = [ { value: 'Light', label: t('settings.general.light') }, { value: 'Dark', label: t('settings.general.dark') }, @@ -46,9 +45,7 @@ export default function General() { [4000, t('settings.general.high')], [1e9, t('settings.general.unlimited')], ]); - const [prompts, setPrompts] = React.useState< - { name: string; id: string; type: string }[] - >([]); + const prompts = useSelector(selectPrompts); const selectedChunks = useSelector(selectChunks); const selectedTokenLimit = useSelector(selectTokenLimit); const [isDarkTheme, toggleTheme] = useDarkTheme(); @@ -64,22 +61,6 @@ export default function General() { ); const selectedPrompt = useSelector(selectPrompt); - React.useEffect(() => { - const handleFetchPrompts = async () => { - try { - const response = await userService.getPrompts(token); - if (!response.ok) { - throw new Error('Failed to fetch prompts'); - } - const promptsData = await response.json(); - setPrompts(promptsData); - } catch (error) { - console.error(error); - } - }; - handleFetchPrompts(); - }, []); - React.useEffect(() => { localStorage.setItem('docsgpt-locale', selectedLanguage?.value as string); changeLanguage(selectedLanguage?.value); @@ -169,7 +150,7 @@ export default function General() { onSelectPrompt={(name, id, type) => dispatch(setPrompt({ name: name, id: id, type: type })) } - setPrompts={setPrompts} + setPrompts={(newPrompts) => dispatch(setPrompts(newPrompts))} dropdownProps={{ size: 'w-56', rounded: '3xl', border: 'border' }} />
diff --git a/frontend/src/settings/Prompts.tsx b/frontend/src/settings/Prompts.tsx index d4de6b42..183b3751 100644 --- a/frontend/src/settings/Prompts.tsx +++ b/frontend/src/settings/Prompts.tsx @@ -3,7 +3,7 @@ import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; import userService from '../api/services/userService'; -import Dropdown from '../components/Dropdown'; +import SearchableDropdown from '../components/SearchableDropdown'; import { DropdownProps } from '../components/types/Dropdown.types'; import ConfirmationModal from '../modals/ConfirmationModal'; import { ActiveState, PromptProps } from '../models/misc'; @@ -103,7 +103,12 @@ export default function Prompts({ if (!response.ok) { throw new Error('Failed to delete prompt'); } - if (prompts.length > 0) { + // Only change selection if we're deleting the currently selected prompt + if ( + prompts.length > 0 && + selectedPrompt && + selectedPrompt.id === promptToDelete.id + ) { const firstPrompt = prompts.find((p) => p.id !== promptToDelete.id); if (firstPrompt) { onSelectPrompt( @@ -182,7 +187,7 @@ export default function Prompts({ {title ? title : t('settings.general.prompt')}

- typeof prompt === 'string' ? { name: prompt, id: prompt, type: '' } diff --git a/frontend/src/store.ts b/frontend/src/store.ts index 45b29916..a3092c82 100644 --- a/frontend/src/store.ts +++ b/frontend/src/store.ts @@ -25,6 +25,11 @@ const preloadedState: { preference: Preference } = { prompt !== null ? JSON.parse(prompt) : { name: 'default', id: 'default', type: 'private' }, + prompts: [ + { name: 'default', id: 'default', type: 'public' }, + { name: 'creative', id: 'creative', type: 'public' }, + { name: 'strict', id: 'strict', type: 'public' }, + ], chunks: JSON.parse(chunks ?? '2').toString(), token_limit: token_limit ? parseInt(token_limit) : 2000, selectedDocs: doc !== null ? JSON.parse(doc) : [], From e0a9f086322d9596dcd46b5bbfa74203ac593db6 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 10 Dec 2025 21:53:59 +0000 Subject: [PATCH 14/93] refactor and deps (#2184) --- .../answer/services/conversation_service.py | 5 +- application/llm/docsgpt_provider.py | 148 +++++------------- application/llm/groq.py | 44 ++---- application/llm/handlers/base.py | 5 +- application/llm/huggingface.py | 68 -------- application/llm/llm_creator.py | 2 - application/llm/novita.py | 41 ++--- application/llm/openai.py | 5 +- application/parser/file/base.py | 2 +- application/parser/remote/base.py | 2 +- application/parser/schema/base.py | 2 +- application/requirements.txt | 90 +++++------ tests/parser/remote/test_crawler_loader.py | 2 +- tests/parser/remote/test_web_loader.py | 2 +- 14 files changed, 120 insertions(+), 298 deletions(-) delete mode 100644 application/llm/huggingface.py diff --git a/application/api/answer/services/conversation_service.py b/application/api/answer/services/conversation_service.py index bf55801c..5d37e32b 100644 --- a/application/api/answer/services/conversation_service.py +++ b/application/api/answer/services/conversation_service.py @@ -148,9 +148,12 @@ class ConversationService: ] completion = llm.gen( - model=model_id, messages=messages_summary, max_tokens=30 + model=model_id, messages=messages_summary, max_tokens=500 ) + if not completion or not completion.strip(): + completion = question[:50] if question else "New Conversation" + conversation_data = { "user": user_id, "date": current_time, diff --git a/application/llm/docsgpt_provider.py b/application/llm/docsgpt_provider.py index 44a479ae..bc52bcfc 100644 --- a/application/llm/docsgpt_provider.py +++ b/application/llm/docsgpt_provider.py @@ -1,75 +1,19 @@ -import json - -from openai import OpenAI - from application.core.settings import settings -from application.llm.base import BaseLLM +from application.llm.openai import OpenAILLM +DOCSGPT_API_KEY = "sk-docsgpt-public" +DOCSGPT_BASE_URL = "https://oai.arc53.com" +DOCSGPT_MODEL = "docsgpt" -class DocsGPTAPILLM(BaseLLM): - - def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): - - super().__init__(*args, **kwargs) - self.api_key = "sk-docsgpt-public" - self.client = OpenAI(api_key=self.api_key, base_url="https://oai.arc53.com") - self.user_api_key = user_api_key - - def _clean_messages_openai(self, messages): - cleaned_messages = [] - for message in messages: - role = message.get("role") - content = message.get("content") - - if role == "model": - role = "assistant" - if role and content is not None: - if isinstance(content, str): - cleaned_messages.append({"role": role, "content": content}) - elif isinstance(content, list): - for item in content: - if "text" in item: - cleaned_messages.append( - {"role": role, "content": item["text"]} - ) - elif "function_call" in item: - cleaned_args = self._remove_null_values( - item["function_call"]["args"] - ) - tool_call = { - "id": item["function_call"]["call_id"], - "type": "function", - "function": { - "name": item["function_call"]["name"], - "arguments": json.dumps(cleaned_args), - }, - } - cleaned_messages.append( - { - "role": "assistant", - "content": None, - "tool_calls": [tool_call], - } - ) - elif "function_response" in item: - cleaned_messages.append( - { - "role": "tool", - "tool_call_id": item["function_response"][ - "call_id" - ], - "content": json.dumps( - item["function_response"]["response"]["result"] - ), - } - ) - else: - raise ValueError( - f"Unexpected content dictionary format: {item}" - ) - else: - raise ValueError(f"Unexpected content type: {type(content)}") - return cleaned_messages +class DocsGPTAPILLM(OpenAILLM): + def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs): + super().__init__( + api_key=DOCSGPT_API_KEY, + user_api_key=user_api_key, + base_url=DOCSGPT_BASE_URL, + *args, + **kwargs, + ) def _raw_gen( self, @@ -79,23 +23,19 @@ class DocsGPTAPILLM(BaseLLM): stream=False, tools=None, engine=settings.AZURE_DEPLOYMENT_NAME, + response_format=None, **kwargs, ): - messages = self._clean_messages_openai(messages) - if tools: - response = self.client.chat.completions.create( - model="docsgpt", - messages=messages, - stream=stream, - tools=tools, - **kwargs, - ) - return response.choices[0] - else: - response = self.client.chat.completions.create( - model="docsgpt", messages=messages, stream=stream, **kwargs - ) - return response.choices[0].message.content + return super()._raw_gen( + baseself, + DOCSGPT_MODEL, + messages, + stream=stream, + tools=tools, + engine=engine, + response_format=response_format, + **kwargs, + ) def _raw_gen_stream( self, @@ -105,34 +45,16 @@ class DocsGPTAPILLM(BaseLLM): stream=True, tools=None, engine=settings.AZURE_DEPLOYMENT_NAME, + response_format=None, **kwargs, ): - messages = self._clean_messages_openai(messages) - if tools: - response = self.client.chat.completions.create( - model="docsgpt", - messages=messages, - stream=stream, - tools=tools, - **kwargs, - ) - else: - response = self.client.chat.completions.create( - model="docsgpt", messages=messages, stream=stream, **kwargs - ) - try: - for line in response: - if ( - len(line.choices) > 0 - and line.choices[0].delta.content is not None - and len(line.choices[0].delta.content) > 0 - ): - yield line.choices[0].delta.content - elif len(line.choices) > 0: - yield line.choices[0] - finally: - if hasattr(response, "close"): - response.close() - - def _supports_tools(self): - return True + return super()._raw_gen_stream( + baseself, + DOCSGPT_MODEL, + messages, + stream=stream, + tools=tools, + engine=engine, + response_format=response_format, + **kwargs, + ) diff --git a/application/llm/groq.py b/application/llm/groq.py index c2ae40ee..9d7c1713 100644 --- a/application/llm/groq.py +++ b/application/llm/groq.py @@ -1,37 +1,15 @@ -from openai import OpenAI - from application.core.settings import settings -from application.llm.base import BaseLLM +from application.llm.openai import OpenAILLM + +GROQ_BASE_URL = "https://api.groq.com/openai/v1" -class GroqLLM(BaseLLM): - def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): - - super().__init__(*args, **kwargs) - self.api_key = api_key or settings.GROQ_API_KEY or settings.API_KEY - self.user_api_key = user_api_key - self.client = OpenAI( - api_key=self.api_key, base_url="https://api.groq.com/openai/v1" +class GroqLLM(OpenAILLM): + def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs): + super().__init__( + api_key=api_key or settings.GROQ_API_KEY or settings.API_KEY, + user_api_key=user_api_key, + base_url=base_url or GROQ_BASE_URL, + *args, + **kwargs, ) - - def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs): - if tools: - response = self.client.chat.completions.create( - model=model, messages=messages, stream=stream, tools=tools, **kwargs - ) - return response.choices[0] - else: - response = self.client.chat.completions.create( - model=model, messages=messages, stream=stream, **kwargs - ) - return response.choices[0].message.content - - def _raw_gen_stream( - self, baseself, model, messages, stream=True, tools=None, **kwargs - ): - response = self.client.chat.completions.create( - model=model, messages=messages, stream=stream, **kwargs - ) - for line in response: - if line.choices[0].delta.content is not None: - yield line.choices[0].delta.content diff --git a/application/llm/handlers/base.py b/application/llm/handlers/base.py index b11654c5..dbc5a879 100644 --- a/application/llm/handlers/base.py +++ b/application/llm/handlers/base.py @@ -833,7 +833,10 @@ class LLMHandler(ABC): if call.name: existing.name = call.name if call.arguments: - existing.arguments += call.arguments + if existing.arguments is None: + existing.arguments = call.arguments + else: + existing.arguments += call.arguments # Preserve thought_signature for Google Gemini 3 models if call.thought_signature: existing.thought_signature = call.thought_signature diff --git a/application/llm/huggingface.py b/application/llm/huggingface.py deleted file mode 100644 index 2fb4a925..00000000 --- a/application/llm/huggingface.py +++ /dev/null @@ -1,68 +0,0 @@ -from application.llm.base import BaseLLM - - -class HuggingFaceLLM(BaseLLM): - - def __init__( - self, - api_key=None, - user_api_key=None, - llm_name="Arc53/DocsGPT-7B", - q=False, - *args, - **kwargs, - ): - global hf - - from langchain.llms import HuggingFacePipeline - - if q: - import torch - from transformers import ( - AutoModelForCausalLM, - AutoTokenizer, - pipeline, - BitsAndBytesConfig, - ) - - tokenizer = AutoTokenizer.from_pretrained(llm_name) - bnb_config = BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type="nf4", - bnb_4bit_compute_dtype=torch.bfloat16, - ) - model = AutoModelForCausalLM.from_pretrained( - llm_name, quantization_config=bnb_config - ) - else: - from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline - - tokenizer = AutoTokenizer.from_pretrained(llm_name) - model = AutoModelForCausalLM.from_pretrained(llm_name) - - super().__init__(*args, **kwargs) - self.api_key = api_key - self.user_api_key = user_api_key - pipe = pipeline( - "text-generation", - model=model, - tokenizer=tokenizer, - max_new_tokens=2000, - device_map="auto", - eos_token_id=tokenizer.eos_token_id, - ) - hf = HuggingFacePipeline(pipeline=pipe) - - def _raw_gen(self, baseself, model, messages, stream=False, **kwargs): - context = messages[0]["content"] - user_question = messages[-1]["content"] - prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n" - - result = hf(prompt) - - return result.content - - def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs): - - raise NotImplementedError("HuggingFaceLLM Streaming is not implemented yet.") diff --git a/application/llm/llm_creator.py b/application/llm/llm_creator.py index 21d653b9..ca39194c 100644 --- a/application/llm/llm_creator.py +++ b/application/llm/llm_creator.py @@ -4,7 +4,6 @@ from application.llm.anthropic import AnthropicLLM from application.llm.docsgpt_provider import DocsGPTAPILLM from application.llm.google_ai import GoogleLLM from application.llm.groq import GroqLLM -from application.llm.huggingface import HuggingFaceLLM from application.llm.llama_cpp import LlamaCpp from application.llm.novita import NovitaLLM from application.llm.openai import AzureOpenAILLM, OpenAILLM @@ -19,7 +18,6 @@ class LLMCreator: "openai": OpenAILLM, "azure_openai": AzureOpenAILLM, "sagemaker": SagemakerAPILLM, - "huggingface": HuggingFaceLLM, "llama.cpp": LlamaCpp, "anthropic": AnthropicLLM, "docsgpt": DocsGPTAPILLM, diff --git a/application/llm/novita.py b/application/llm/novita.py index 8d6ac042..b741c4f3 100644 --- a/application/llm/novita.py +++ b/application/llm/novita.py @@ -1,32 +1,15 @@ -from application.llm.base import BaseLLM -from openai import OpenAI +from application.core.settings import settings +from application.llm.openai import OpenAILLM + +NOVITA_BASE_URL = "https://api.novita.ai/v3/openai" -class NovitaLLM(BaseLLM): - def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): - super().__init__(*args, **kwargs) - self.client = OpenAI(api_key=api_key, base_url="https://api.novita.ai/v3/openai") - self.api_key = api_key - self.user_api_key = user_api_key - - def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs): - if tools: - response = self.client.chat.completions.create( - model=model, messages=messages, stream=stream, tools=tools, **kwargs - ) - return response.choices[0] - else: - response = self.client.chat.completions.create( - model=model, messages=messages, stream=stream, **kwargs - ) - return response.choices[0].message.content - - def _raw_gen_stream( - self, baseself, model, messages, stream=True, tools=None, **kwargs - ): - response = self.client.chat.completions.create( - model=model, messages=messages, stream=stream, **kwargs +class NovitaLLM(OpenAILLM): + def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs): + super().__init__( + api_key=api_key or settings.API_KEY, + user_api_key=user_api_key, + base_url=base_url or NOVITA_BASE_URL, + *args, + **kwargs, ) - for line in response: - if line.choices[0].delta.content is not None: - yield line.choices[0].delta.content diff --git a/application/llm/openai.py b/application/llm/openai.py index 3917cbf7..e851f078 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -127,6 +127,7 @@ class OpenAILLM(BaseLLM): **kwargs, ): messages = self._clean_messages_openai(messages) + logging.info(f"Cleaned messages: {messages}") # Convert max_tokens to max_completion_tokens for newer models if "max_tokens" in kwargs: @@ -144,7 +145,7 @@ class OpenAILLM(BaseLLM): if response_format: request_params["response_format"] = response_format response = self.client.chat.completions.create(**request_params) - + logging.info(f"OpenAI response: {response}") if tools: return response.choices[0] else: @@ -162,6 +163,7 @@ class OpenAILLM(BaseLLM): **kwargs, ): messages = self._clean_messages_openai(messages) + logging.info(f"Cleaned messages: {messages}") # Convert max_tokens to max_completion_tokens for newer models if "max_tokens" in kwargs: @@ -182,6 +184,7 @@ class OpenAILLM(BaseLLM): try: for line in response: + logging.debug(f"OpenAI stream line: {line}") if ( len(line.choices) > 0 and line.choices[0].delta.content is not None diff --git a/application/parser/file/base.py b/application/parser/file/base.py index f63e8ef6..8e9b1015 100644 --- a/application/parser/file/base.py +++ b/application/parser/file/base.py @@ -2,7 +2,7 @@ from abc import abstractmethod from typing import Any, List -from langchain.docstore.document import Document as LCDocument +from langchain_core.documents import Document as LCDocument from application.parser.schema.base import Document diff --git a/application/parser/remote/base.py b/application/parser/remote/base.py index 91313f22..74b6fce7 100644 --- a/application/parser/remote/base.py +++ b/application/parser/remote/base.py @@ -2,7 +2,7 @@ from abc import abstractmethod from typing import Any, List -from langchain.docstore.document import Document as LCDocument +from langchain_core.documents import Document as LCDocument from application.parser.schema.base import Document diff --git a/application/parser/schema/base.py b/application/parser/schema/base.py index 61670f9a..a7453dd7 100644 --- a/application/parser/schema/base.py +++ b/application/parser/schema/base.py @@ -1,7 +1,7 @@ """Base schema for readers.""" from dataclasses import dataclass -from langchain.docstore.document import Document as LCDocument +from langchain_core.documents import Document as LCDocument from application.parser.schema.schema import BaseDocument diff --git a/application/requirements.txt b/application/requirements.txt index cb58247b..ca8be5cc 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -1,91 +1,91 @@ -anthropic==0.49.0 -boto3==1.38.18 -beautifulsoup4==4.13.4 -celery==5.4.0 -cryptography==42.0.8 +anthropic==0.75.0 +boto3==1.42.6 +beautifulsoup4==4.14.3 +celery==5.6.0 +cryptography==46.0.3 dataclasses-json==0.6.7 docx2txt==0.8 -duckduckgo-search==7.5.2 +duckduckgo-search==8.1.1 ebooklib==0.18 escodegen==1.0.11 esprima==4.0.1 esutils==1.0.1 -elevenlabs==2.17.0 -Flask==3.1.1 -faiss-cpu==1.9.0.post1 -fastmcp==2.11.0 -flask-restx==1.3.0 -google-genai==1.49.0 -google-api-python-client==2.179.0 +elevenlabs==2.26.1 +Flask==3.1.2 +faiss-cpu==1.13.1 +fastmcp==2.13.3 +flask-restx==1.3.2 +google-genai==1.54.0 +google-api-python-client==2.187.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.2 gTTS==2.5.4 gunicorn==23.0.0 javalang==0.13.0 jinja2==3.1.6 -jiter==0.8.2 +jiter==0.12.0 jmespath==1.0.1 joblib==1.4.2 jsonpatch==1.33 jsonpointer==3.0.0 -kombu==5.4.2 -langchain==0.3.20 -langchain-community==0.3.19 -langchain-core==0.3.59 -langchain-openai==0.3.16 -langchain-text-splitters==0.3.8 -langsmith==0.3.42 +kombu==5.6.1 +langchain==1.1.3 +langchain-community==0.4.1 +langchain-core==1.1.3 +langchain-openai==1.1.1 +langchain-text-splitters==1.0.0 +langsmith==0.4.58 lazy-object-proxy==1.10.0 -lxml==5.3.1 +lxml==6.0.2 markupsafe==3.0.2 marshmallow==3.26.1 mpmath==1.3.0 -multidict==6.4.3 +multidict==6.7.0 mypy-extensions==1.0.0 -networkx==3.4.2 +networkx==3.6.1 numpy==2.2.1 -openai==1.78.1 +openai==2.9.0 openapi3-parser==1.1.21 -orjson==3.10.14 +orjson==3.11.5 packaging==24.2 pandas==2.2.3 openpyxl==3.1.5 pathable==0.4.4 -pillow==11.1.0 +pillow==12.0.0 portalocker>=2.7.0,<3.0.0 prance==23.6.21.0 prompt-toolkit==3.0.51 protobuf==5.29.3 -psycopg2-binary==2.9.10 +psycopg2-binary==2.9.11 py==1.11.0 pydantic pydantic-core pydantic-settings -pymongo==4.11.3 -pypdf==5.5.0 +pymongo==4.15.5 +pypdf==6.4.1 python-dateutil==2.9.0.post0 python-dotenv python-jose==3.4.0 python-pptx==1.0.2 -redis==5.2.1 +redis==7.1.0 referencing>=0.28.0,<0.31.0 -regex==2024.11.6 -requests==2.32.3 +regex==2025.11.3 +requests==2.32.5 retry==0.9.2 -sentence-transformers==3.3.1 -tiktoken==0.8.0 -tokenizers==0.21.0 +sentence-transformers==5.1.2 +tiktoken==0.12.0 +tokenizers==0.22.1 torch==2.7.0 tqdm==4.67.1 -transformers==4.51.3 -typing-extensions==4.12.2 +transformers==4.57.3 +typing-extensions==4.15.0 typing-inspect==0.9.0 -tzdata==2024.2 -urllib3==2.3.0 +tzdata==2025.2 +urllib3==2.6.1 vine==5.1.0 wcwidth==0.2.13 -werkzeug>=3.1.0,<3.1.2 -yarl==1.20.0 -markdownify==1.1.0 -tldextract==5.1.3 -websockets==14.1 +werkzeug>=3.1.0 +yarl==1.22.0 +markdownify==1.2.2 +tldextract==5.3.0 +websockets==15.0.1 \ No newline at end of file diff --git a/tests/parser/remote/test_crawler_loader.py b/tests/parser/remote/test_crawler_loader.py index 0a100abb..92ffdc84 100644 --- a/tests/parser/remote/test_crawler_loader.py +++ b/tests/parser/remote/test_crawler_loader.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch from application.parser.remote.crawler_loader import CrawlerLoader from application.parser.schema.base import Document -from langchain.docstore.document import Document as LCDocument +from langchain_core.documents import Document as LCDocument class DummyResponse: diff --git a/tests/parser/remote/test_web_loader.py b/tests/parser/remote/test_web_loader.py index ca539f0a..73e368a5 100644 --- a/tests/parser/remote/test_web_loader.py +++ b/tests/parser/remote/test_web_loader.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse from application.parser.remote.web_loader import WebLoader, headers from application.parser.schema.base import Document -from langchain.docstore.document import Document as LCDocument +from langchain_core.documents import Document as LCDocument @pytest.fixture From d14f04d79c1da208a12cc325885599101a929dd8 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 11 Dec 2025 00:54:58 +0200 Subject: [PATCH 15/93] Update requirements.txt --- application/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/requirements.txt b/application/requirements.txt index ca8be5cc..c02f75c0 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -6,7 +6,7 @@ cryptography==46.0.3 dataclasses-json==0.6.7 docx2txt==0.8 duckduckgo-search==8.1.1 -ebooklib==0.18 +ebooklib==0.20 escodegen==1.0.11 esprima==4.0.1 esutils==1.0.1 @@ -88,4 +88,4 @@ werkzeug>=3.1.0 yarl==1.22.0 markdownify==1.2.2 tldextract==5.3.0 -websockets==15.0.1 \ No newline at end of file +websockets==15.0.1 From 909bc421c07d48b857f8287b83e9c6f0dcc02112 Mon Sep 17 00:00:00 2001 From: Mohamed-Abuali Date: Wed, 10 Dec 2025 19:35:55 -0600 Subject: [PATCH 16/93] Bugfix/docs gpt widget behavior (#2172) * style(DocsGPTWidget): improve message bubbles and markdown styling - Adjust max-width for message bubbles to 90% for answers and 80% for questions - Add overflow-wrap to prevent text overflow in messages - Update list styling with proper spacing and positioning - Add responsive font sizing for headings using clamp() - Implement custom table styling with proper borders and spacing - Add custom markdown renderer rules for tables * feat(widget): replace input with textarea for prompt input Add support for multi-line input and custom scrollbar styling. Implement Enter key submission handling while allowing Shift+Enter for new lines. * feat(widget): improve textarea auto-resizing and table styling - Add auto-resizing functionality for prompt textarea with min/max height constraints - Fix table cell markup (th/td) and improve scrollbar styling - Add promptRef to manage textarea state and reset after submission * fix(widget): correct table cell styling and prevent empty submissions - Fix swapped td/th elements in markdown renderer - Adjust font weights for table headers and cells - Add validation to prevent empty message submissions * (fix) name mkdwn rule as the returned element --------- Co-authored-by: ManishMadan2882 --- .../src/components/DocsGPTWidget.tsx | 192 ++++++++++++++++-- 1 file changed, 177 insertions(+), 15 deletions(-) diff --git a/extensions/react-widget/src/components/DocsGPTWidget.tsx b/extensions/react-widget/src/components/DocsGPTWidget.tsx index 5ce9140a..7c16311e 100644 --- a/extensions/react-widget/src/components/DocsGPTWidget.tsx +++ b/extensions/react-widget/src/components/DocsGPTWidget.tsx @@ -293,6 +293,8 @@ const MessageBubble = styled.div<{ type: MESSAGE_TYPE }>` margin: 0px; &:hover ${Feedback} * { visibility: visible ; + + } `; const Message = styled.div<{ type: MESSAGE_TYPE }>` @@ -302,13 +304,14 @@ const Message = styled.div<{ type: MESSAGE_TYPE }>` color: ${props => props.type === 'ANSWER' ? props.theme.primary.text : '#fff'}; border: none; float: ${props => props.type === 'QUESTION' ? 'right' : 'left'}; - max-width: ${props => props.type === 'ANSWER' ? '100%' : '80'}; + max-width: ${props => props.type === 'ANSWER' ? '90%' : '80%'}; overflow: auto; margin: 4px; display: block; line-height: 1.5; padding: 12px; border-radius: 6px; + overflow-wrap: break-word; `; const Markdown = styled.div` pre { @@ -322,7 +325,7 @@ const Markdown = styled.div` } h1 { - font-size: 16px; + font-size: clamp(14px,40vw,16px); } h2 { @@ -334,6 +337,7 @@ const Markdown = styled.div` } p { + margin: 0px; } @@ -354,8 +358,67 @@ const Markdown = styled.div` ul{ padding:0px; - list-style-position: inside; + margin: 1rem 0; + list-style-position: outside; + list-style-type: disc; + padding-left: 1rem; + white-space: normal; } + + ol{ + padding:0px; + margin: 1rem 0; + list-style-position: outside; + list-style-type: decimal; + padding-left: 1rem; + white-space: normal; + } + + li{ + line-height: 1.625; + } + .dgpt-table-container { + margin: 20px 0; + width:100%; + overflow-x: scroll !important; + border: 1px solid #a2a2ab; + border-radius: 6px; + -webkit-overflow-scrolling: touch; + -ms-overflow-style: scrollbar; + scrollbar-width: thin; + scrollbar-color: #a2a2ab #38383b; + } + + + table, .dgpt-table { + width: 100%; + border-collapse: collapse; + text-align: left; + min-width:600px; + + } + thead, .dgpt-thead { + font-size: 12px; + text-transform: uppercase; + + } + + + th, .dgpt-th, td, .dgpt-td { + padding: 10px; + border-bottom: 1px solid #a2a2ab; + font-size:14px; + + } + th{ + font-weight: normal !important; + } + td{ + font-weight: bold; + } + + + ` const ErrorAlert = styled.div` color: #b91c1c; @@ -389,19 +452,49 @@ const Delay = styled(DotAnimation) <{ delay: number }>` `; const PromptContainer = styled.form` background-color: transparent; - height: ${props => props.theme.dimensions.size == 'large' ? '60px' : '40px'}; + min-height: ${props => props.theme.dimensions.size == 'large' ? '40px' : '23px'}; + max-height:150px; display: flex; + align-items: end; justify-content: space-evenly; `; -const StyledInput = styled.input` +const StyledTextarea = styled.textarea` + box-sizing: border-box; width: 100%; border: 1px solid #686877; - padding-left: 12px; + padding: ${props => props.theme.dimensions.size === 'large' ? '18px 12px 14px 12px' : '8px 12px 4px 12px'}; background-color: transparent; font-size: 16px; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; border-radius: 6px; color: ${props => props.theme.text}; outline: none; + resize: none; + transition: height 0.1s ease; + overflow-wrap: break-word; + white-space: pre-wrap; + line-height: 1.4; + text-align: left; + min-height: ${props => props.theme.dimensions.size === 'large' ? '60px' : '40px'}; + max-height: 140px; + overflow-y: auto; + scrollbar-width: thin; + scrollbar-color: #38383b transparent; + &::-webkit-scrollbar { + width: 6px; + height: 6px; + } + &::-webkit-scrollbar-thumb { + background-color: #38383b; + border-radius: 6px; + } + &::-webkit-scrollbar-track { + background: transparent; + } + &::placeholder { + text-align: left; + + } `; const StyledButton = styled.button` display: flex; @@ -619,7 +712,17 @@ export const WidgetCore = ({ const isBubbleHovered = useRef(false); const conversationRef = useRef(null); const endMessageRef = React.useRef(null); + const promptRef = React.useRef(null); const md = new MarkdownIt(); + //Custom markdown for the table + md.renderer.rules.table_open = () => '
'; + md.renderer.rules.table_close = () => '
'; + md.renderer.rules.thead_open = () => ''; + md.renderer.rules.tr_open = () => ''; + md.renderer.rules.td_open = () => ''; + md.renderer.rules.th_open = () => ''; + + React.useEffect(() => { if (isOpen) { @@ -774,11 +877,7 @@ export const WidgetCore = ({ } } - // submit handler - const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - await appendQuery(prompt) - } + const appendQuery = async (userQuery: string) => { if (!userQuery) @@ -788,6 +887,58 @@ export const WidgetCore = ({ queries.push({ prompt: userQuery }); setPrompt(''); await stream(userQuery); + } + // submit handler + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + if (!prompt.trim()) return; + if (promptRef.current) { + promptRef.current.style.height = "auto"; + } + await appendQuery(prompt); + } + const handlePromptKeyDown = async (e: React.KeyboardEvent) => { + if (e.key === 'Enter' && !e.shiftKey) { + + e.preventDefault(); + // Prevent sending empty messages + if (promptRef.current && promptRef.current.value.trim() === "") return; + //Rest the input to it's original size after submitting + if(promptRef.current){ + promptRef.current.value = ""; + promptRef.current.style.height = "auto"; + } + await appendQuery(prompt); + } +} + // Auto-resize the input textarea while typing, clamping to base or max height + const handleUserInput = (e: React.KeyboardEvent) =>{ + const el = promptRef.current; + if (!el) return; + const baseHeight = size === 'large' ? 60 : 40; + const maxHeight = 140; + el.style.height = 'auto'; + const next = Math.min(el.scrollHeight, maxHeight); + el.style.height = Math.max(baseHeight, next) + 'px'; + + } + + // Update prompt state, auto resize textarea to content, and maintain scroll on new lines + const handlePromptChange = (event: React.ChangeEvent) => { + const value = event.target.value; + setPrompt(value); + const el = event.currentTarget; + const baseHeight = size === 'large' ? 60 : 40; + const maxHeight = 140; + el.style.height = 'auto'; + const next = Math.min(el.scrollHeight, maxHeight); + el.style.height = Math.max(baseHeight, next) + 'px'; + if(value.includes("\n")){ + el.scrollTop = el.scrollHeight; + + } + + } const handleImageError = (event: React.SyntheticEvent) => { event.currentTarget.src = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png"; @@ -798,6 +949,9 @@ export const WidgetCore = ({ ? sizesConfig.getCustom(size.custom) : sizesConfig[size]; if (!mounted) return null; + + + return ( {isOpen && size === 'large' && @@ -911,10 +1065,18 @@ export const WidgetCore = ({
- setPrompt(event.target.value)} - type='text' placeholder="Ask your question" /> + onInput={handleUserInput} + value={prompt} + onChange={handlePromptChange} + placeholder="Ask your question" + onKeyDown={handlePromptKeyDown} + rows={1} + wrap="soft" + /> @@ -931,4 +1093,4 @@ export const WidgetCore = ({ } ) -} +} \ No newline at end of file From 6d8f083c6f9650a65331ade64fe2ca256a60f2aa Mon Sep 17 00:00:00 2001 From: AbbasSalloum <63022908+AbbasSalloum@users.noreply.github.com> Date: Fri, 12 Dec 2025 06:55:16 -0500 Subject: [PATCH 17/93] Adding a feature to paste files you ctrl v (#2183) --- frontend/src/components/MessageInput.tsx | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/frontend/src/components/MessageInput.tsx b/frontend/src/components/MessageInput.tsx index 4e494959..750b0b12 100644 --- a/frontend/src/components/MessageInput.tsx +++ b/frontend/src/components/MessageInput.tsx @@ -532,6 +532,31 @@ export default function MessageInput({ } }; + const handlePaste = (e: React.ClipboardEvent) => { + const clipboardItems = e.clipboardData?.items; + const files: File[] = []; + + if (!clipboardItems) return; + + for (let i = 0; i < clipboardItems.length; i++) { + const item = clipboardItems[i]; + + if (item.kind === 'file') { + const file = item.getAsFile(); + if (file) { + files.push(file); + } + } + } + + if (files.length > 0) { + // Prevent weird binary stuff from being pasted as text + e.preventDefault(); + uploadFiles(files); + } +}; + + const handlePostDocumentSelect = (doc: any) => { console.log('Selected document:', doc); }; @@ -691,6 +716,7 @@ export default function MessageInput({ className="inputbox-style no-scrollbar bg-lotion dark:text-bright-gray dark:placeholder:text-bright-gray/50 w-full overflow-x-hidden overflow-y-auto rounded-t-[23px] px-2 text-base leading-tight whitespace-pre-wrap opacity-100 placeholder:text-gray-500 focus:outline-hidden sm:px-3 dark:bg-transparent" onInput={handleInput} onKeyDown={handleKeyDown} + onPaste={handlePaste} aria-label={t('inputPlaceholder')} />
From aacf28122259676389a98ac1d2cf69b3568e30b1 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 16 Dec 2025 11:59:17 +0000 Subject: [PATCH 18/93] fix: improve remote embeds (#2193) --- application/vectorstore/base.py | 36 ++++++++++++++--------------- application/vectorstore/pgvector.py | 16 ++++++------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/application/vectorstore/base.py b/application/vectorstore/base.py index e5c65794..b3cf4da4 100644 --- a/application/vectorstore/base.py +++ b/application/vectorstore/base.py @@ -12,6 +12,7 @@ class RemoteEmbeddings: """ Wrapper for remote embeddings API (OpenAI-compatible). Used when EMBEDDINGS_BASE_URL is configured. + Sends requests to {base_url}/v1/embeddings in OpenAI format. """ def __init__(self, api_url: str, model_name: str, api_key: str = None): @@ -20,33 +21,30 @@ class RemoteEmbeddings: self.headers = {"Content-Type": "application/json"} if api_key: self.headers["Authorization"] = f"Bearer {api_key}" - self.dimension = None + self.dimension = 768 def _embed(self, inputs): - """Send embedding request to remote API.""" - payload = {"inputs": inputs} + """Send embedding request to remote API in OpenAI-compatible format.""" + payload = {"input": inputs} if self.model_name: payload["model"] = self.model_name - response = requests.post( - self.api_url, headers=self.headers, json=payload, timeout=180 - ) + url = f"{self.api_url}/v1/embeddings" + response = requests.post(url, headers=self.headers, json=payload, timeout=180) response.raise_for_status() result = response.json() - if isinstance(result, list): - if result and isinstance(result[0], list): - return result - elif result and all(isinstance(x, (int, float)) for x in result): - return [result] - elif not result: - return [] - else: - raise ValueError( - f"Unexpected list content from remote embeddings API: {result}" - ) - elif isinstance(result, dict) and "error" in result: - raise ValueError(f"Remote embeddings API error: {result['error']}") + # Handle OpenAI-compatible response format + if isinstance(result, dict): + if "error" in result: + raise ValueError(f"Remote embeddings API error: {result['error']}") + if "data" in result: + # Sort by index to ensure correct order + data = sorted(result["data"], key=lambda x: x.get("index", 0)) + return [item["embedding"] for item in data] + raise ValueError( + f"Unexpected response format from remote embeddings API: {result}" + ) else: raise ValueError( f"Unexpected response format from remote embeddings API: {result}" diff --git a/application/vectorstore/pgvector.py b/application/vectorstore/pgvector.py index 6a7547b3..28233821 100644 --- a/application/vectorstore/pgvector.py +++ b/application/vectorstore/pgvector.py @@ -11,6 +11,7 @@ class PGVectorStore(BaseVectorStore): source_id: str = "", embeddings_key: str = "embeddings", table_name: str = "documents", + decoded_token: Optional[str] = None, vector_column: str = "embedding", text_column: str = "text", metadata_column: str = "metadata", @@ -68,8 +69,7 @@ class PGVectorStore(BaseVectorStore): # Enable pgvector extension cursor.execute("CREATE EXTENSION IF NOT EXISTS vector;") - # Get embedding dimension - embedding_dim = getattr(self._embedding, 'dimension', 1536) # Default to OpenAI dimension + embedding_dim = getattr(self._embedding, 'dimension', 768) # Create table with vector column create_table_query = f""" @@ -152,7 +152,7 @@ class PGVectorStore(BaseVectorStore): """Add texts with their embeddings to the vector store""" if not texts: return [] - + embeddings = self._embedding.embed_documents(texts) metadatas = metadatas or [{}] * len(texts) @@ -239,15 +239,13 @@ class PGVectorStore(BaseVectorStore): def add_chunk(self, text: str, metadata: Optional[Dict[str, Any]] = None) -> str: """Add a single chunk to the vector store""" metadata = metadata or {} - - # Create a copy to avoid modifying the original metadata + final_metadata = metadata.copy() - - # Ensure the source_id is in the metadata so the chunk can be found by filters + final_metadata["source_id"] = self._source_id - + embeddings = self._embedding.embed_documents([text]) - + if not embeddings: raise ValueError("Could not generate embedding for chunk") From af3e16c4fcc1fb96717e0208d2dffbd3a9a6ebf2 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 17 Dec 2025 01:34:17 +0000 Subject: [PATCH 19/93] fix: count history tokens from chunks, remove old UI setting limit (#2196) --- application/api/answer/routes/answer.py | 1 - application/api/answer/routes/stream.py | 1 - .../api/answer/services/stream_processor.py | 4 +-- application/utils.py | 4 +-- frontend/src/agents/agentPreviewSlice.ts | 2 -- .../src/conversation/conversationHandlers.ts | 6 ---- .../src/conversation/conversationModels.ts | 1 - .../src/conversation/conversationSlice.ts | 2 -- frontend/src/preferences/preferenceSlice.ts | 20 ----------- frontend/src/settings/General.tsx | 36 ------------------- frontend/src/store.ts | 2 -- tests/api/conftest.py | 1 - 12 files changed, 3 insertions(+), 77 deletions(-) diff --git a/application/api/answer/routes/answer.py b/application/api/answer/routes/answer.py index bc7ec58c..e79ea378 100644 --- a/application/api/answer/routes/answer.py +++ b/application/api/answer/routes/answer.py @@ -40,7 +40,6 @@ class AnswerResource(Resource, BaseAnswerResource): "chunks": fields.Integer( required=False, default=2, description="Number of chunks" ), - "token_limit": fields.Integer(required=False, description="Token limit"), "retriever": fields.String(required=False, description="Retriever type"), "api_key": fields.String(required=False, description="API key"), "active_docs": fields.String( diff --git a/application/api/answer/routes/stream.py b/application/api/answer/routes/stream.py index b2827a93..7583aa0b 100644 --- a/application/api/answer/routes/stream.py +++ b/application/api/answer/routes/stream.py @@ -40,7 +40,6 @@ class StreamResource(Resource, BaseAnswerResource): "chunks": fields.Integer( required=False, default=2, description="Number of chunks" ), - "token_limit": fields.Integer(required=False, description="Token limit"), "retriever": fields.String(required=False, description="Retriever type"), "api_key": fields.String(required=False, description="API key"), "active_docs": fields.String( diff --git a/application/api/answer/services/stream_processor.py b/application/api/answer/services/stream_processor.py index 912aff65..cf5fc1f9 100644 --- a/application/api/answer/services/stream_processor.py +++ b/application/api/answer/services/stream_processor.py @@ -420,16 +420,14 @@ class StreamProcessor: ) def _configure_retriever(self): - history_token_limit = int(self.data.get("token_limit", 2000)) doc_token_limit = calculate_doc_token_budget( - model_id=self.model_id, history_token_limit=history_token_limit + model_id=self.model_id ) self.retriever_config = { "retriever_name": self.data.get("retriever", "classic"), "chunks": int(self.data.get("chunks", 2)), "doc_token_limit": doc_token_limit, - "history_token_limit": history_token_limit, } api_key = self.data.get("api_key") or self.agent_key diff --git a/application/utils.py b/application/utils.py index b25c4717..35b61036 100644 --- a/application/utils.py +++ b/application/utils.py @@ -77,11 +77,11 @@ def count_tokens_docs(docs): def calculate_doc_token_budget( - model_id: str = "gpt-4o", history_token_limit: int = 2000 + model_id: str = "gpt-4o" ) -> int: total_context = get_token_limit(model_id) reserved = sum(settings.RESERVED_TOKENS.values()) - doc_budget = total_context - history_token_limit - reserved + doc_budget = total_context - reserved return max(doc_budget, 1000) diff --git a/frontend/src/agents/agentPreviewSlice.ts b/frontend/src/agents/agentPreviewSlice.ts index 3e601229..d765b789 100644 --- a/frontend/src/agents/agentPreviewSlice.ts +++ b/frontend/src/agents/agentPreviewSlice.ts @@ -65,7 +65,6 @@ export const fetchPreviewAnswer = createAsyncThunk< null, // No conversation ID for previews state.preference.prompt.id, state.preference.chunks, - state.preference.token_limit, (event: MessageEvent) => { const data = JSON.parse(event.data); const targetIndex = indx ?? state.agentPreview.queries.length - 1; @@ -136,7 +135,6 @@ export const fetchPreviewAnswer = createAsyncThunk< null, state.preference.prompt.id, state.preference.chunks, - state.preference.token_limit, state.preference.selectedAgent?.id, attachmentIds, false, diff --git a/frontend/src/conversation/conversationHandlers.ts b/frontend/src/conversation/conversationHandlers.ts index b6222150..e55952fe 100644 --- a/frontend/src/conversation/conversationHandlers.ts +++ b/frontend/src/conversation/conversationHandlers.ts @@ -11,7 +11,6 @@ export function handleFetchAnswer( conversationId: string | null, promptId: string | null, chunks: string, - token_limit: number, agentId?: string, attachments?: string[], save_conversation = true, @@ -42,7 +41,6 @@ export function handleFetchAnswer( conversation_id: conversationId, prompt_id: promptId, chunks: chunks, - token_limit: token_limit, isNoneDoc: selectedDocs.length === 0, agent_id: agentId, save_conversation: save_conversation, @@ -100,7 +98,6 @@ export function handleFetchAnswerSteaming( conversationId: string | null, promptId: string | null, chunks: string, - token_limit: number, onEvent: (event: MessageEvent) => void, indx?: number, agentId?: string, @@ -113,7 +110,6 @@ export function handleFetchAnswerSteaming( conversation_id: conversationId, prompt_id: promptId, chunks: chunks, - token_limit: token_limit, isNoneDoc: selectedDocs.length === 0, index: indx, agent_id: agentId, @@ -198,13 +194,11 @@ export function handleSearch( selectedDocs: Doc[], conversation_id: string | null, chunks: string, - token_limit: number, ) { const payload: RetrievalPayload = { question: question, conversation_id: conversation_id, chunks: chunks, - token_limit: token_limit, isNoneDoc: selectedDocs.length === 0, }; if (selectedDocs.length > 0) { diff --git a/frontend/src/conversation/conversationModels.ts b/frontend/src/conversation/conversationModels.ts index cbcb644e..904683a7 100644 --- a/frontend/src/conversation/conversationModels.ts +++ b/frontend/src/conversation/conversationModels.ts @@ -59,7 +59,6 @@ export interface RetrievalPayload { conversation_id: string | null; prompt_id?: string | null; chunks: string; - token_limit: number; isNoneDoc: boolean; index?: number; agent_id?: string; diff --git a/frontend/src/conversation/conversationSlice.ts b/frontend/src/conversation/conversationSlice.ts index 8b4eea0e..0298d78c 100644 --- a/frontend/src/conversation/conversationSlice.ts +++ b/frontend/src/conversation/conversationSlice.ts @@ -63,7 +63,6 @@ export const fetchAnswer = createAsyncThunk< currentConversationId, state.preference.prompt.id, state.preference.chunks, - state.preference.token_limit, (event) => { const data = JSON.parse(event.data); const targetIndex = indx ?? state.conversation.queries.length - 1; @@ -171,7 +170,6 @@ export const fetchAnswer = createAsyncThunk< state.conversation.conversationId, state.preference.prompt.id, state.preference.chunks, - state.preference.token_limit, state.preference.selectedAgent?.id, attachmentIds, true, diff --git a/frontend/src/preferences/preferenceSlice.ts b/frontend/src/preferences/preferenceSlice.ts index 2bd5faea..89239b30 100644 --- a/frontend/src/preferences/preferenceSlice.ts +++ b/frontend/src/preferences/preferenceSlice.ts @@ -21,7 +21,6 @@ export interface Preference { prompt: { name: string; id: string; type: string }; prompts: Prompt[]; chunks: string; - token_limit: number; selectedDocs: Doc[]; sourceDocs: Doc[] | null; conversations: { @@ -49,7 +48,6 @@ const initialState: Preference = { { name: 'strict', id: 'strict', type: 'public' }, ], chunks: '2', - token_limit: 2000, selectedDocs: [ { id: 'default', @@ -108,9 +106,6 @@ export const prefSlice = createSlice({ setChunks: (state, action) => { state.chunks = action.payload; }, - setTokenLimit: (state, action) => { - state.token_limit = action.payload; - }, setModalStateDeleteConv: (state, action: PayloadAction) => { state.modalState = action.payload; }, @@ -147,7 +142,6 @@ export const { setPrompt, setPrompts, setChunks, - setTokenLimit, setModalStateDeleteConv, setPaginatedDocuments, setTemplateAgents, @@ -200,18 +194,6 @@ prefListenerMiddleware.startListening({ }, }); -prefListenerMiddleware.startListening({ - matcher: isAnyOf(setTokenLimit), - effect: (action, listenerApi) => { - localStorage.setItem( - 'DocsGPTTokenLimit', - JSON.stringify( - (listenerApi.getState() as RootState).preference.token_limit, - ), - ); - }, -}); - prefListenerMiddleware.startListening({ matcher: isAnyOf(setSourceDocs), effect: (_action, listenerApi) => { @@ -281,8 +263,6 @@ export const selectToken = (state: RootState) => state.preference.token; export const selectPrompt = (state: RootState) => state.preference.prompt; export const selectPrompts = (state: RootState) => state.preference.prompts; export const selectChunks = (state: RootState) => state.preference.chunks; -export const selectTokenLimit = (state: RootState) => - state.preference.token_limit; export const selectPaginatedDocuments = (state: RootState) => state.preference.paginatedDocuments; export const selectTemplateAgents = (state: RootState) => diff --git a/frontend/src/settings/General.tsx b/frontend/src/settings/General.tsx index 8b2c53cc..74479832 100644 --- a/frontend/src/settings/General.tsx +++ b/frontend/src/settings/General.tsx @@ -8,12 +8,10 @@ import { selectChunks, selectPrompt, selectPrompts, - selectTokenLimit, setChunks, setModalStateDeleteConv, setPrompt, setPrompts, - setTokenLimit, } from '../preferences/preferenceSlice'; import Prompts from './Prompts'; @@ -37,17 +35,8 @@ export default function General() { { label: 'Русский', value: 'ru' }, ]; const chunks = ['0', '2', '4', '6', '8', '10']; - const token_limits = new Map([ - [0, t('settings.general.none')], - [100, t('settings.general.low')], - [1000, t('settings.general.medium')], - [2000, t('settings.general.default')], - [4000, t('settings.general.high')], - [1e9, t('settings.general.unlimited')], - ]); const prompts = useSelector(selectPrompts); const selectedChunks = useSelector(selectChunks); - const selectedTokenLimit = useSelector(selectTokenLimit); const [isDarkTheme, toggleTheme] = useDarkTheme(); const [selectedTheme, setSelectedTheme] = React.useState( isDarkTheme ? 'Dark' : 'Light', @@ -118,31 +107,6 @@ export default function General() { border="border" />
-
- - ({ - value: value, - description: desc, - }))} - selectedValue={{ - value: selectedTokenLimit, - description: token_limits.get(selectedTokenLimit) as string, - }} - onSelect={({ - value, - description, - }: { - value: number; - description: string; - }) => dispatch(setTokenLimit(value))} - size="w-56" - rounded="3xl" - border="border" - /> -
Date: Wed, 17 Dec 2025 11:07:50 +0000 Subject: [PATCH 20/93] fix: history leftover (#2197) --- application/worker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/worker.py b/application/worker.py index cbc0dfdc..8442e7f4 100755 --- a/application/worker.py +++ b/application/worker.py @@ -190,9 +190,8 @@ def run_agent_logic(agent_config, input_data): system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER) # Calculate proper doc_token_limit based on model's context window - history_token_limit = 2000 # Default for webhooks doc_token_limit = calculate_doc_token_budget( - model_id=model_id, history_token_limit=history_token_limit + model_id=model_id ) retriever = RetrieverCreator.create_retriever( From e0fd11a86e1796f39108c29f506d18049c339b25 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 17 Dec 2025 14:06:53 +0000 Subject: [PATCH 21/93] fix: bump next --- docs/package-lock.json | 1511 ++++++++-------------------------------- docs/package.json | 2 +- 2 files changed, 302 insertions(+), 1211 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index e1aa2584..daaa687c 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -8,7 +8,7 @@ "dependencies": { "@vercel/analytics": "^1.1.1", "docsgpt-react": "^0.5.1", - "next": "^15.3.3", + "next": "^15.5.9", "nextra": "^2.13.2", "nextra-theme-docs": "^2.13.2", "react": "^18.2.0", @@ -349,9 +349,9 @@ "integrity": "sha512-s3jaWicZd0pkP0jf5ysyHUI/RE7MHos6qlToFcGWXVp+ykHOy77OUMrfbgJ9it2C5bow7OIQwYYaHjk9XlBQ2A==" }, "node_modules/@emnapi/runtime": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.3.tgz", - "integrity": "sha512-pBPWdu6MLKROBX05wSNKcNb++m5Er+KQ9QkB+WVM+pW2Kx9hoSrVTnu3BdkI5eBLZoKu/J6mW/B6i6bJB2ytXQ==", + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz", + "integrity": "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==", "license": "MIT", "optional": true, "dependencies": { @@ -391,10 +391,20 @@ "react-dom": "^16 || ^17 || ^18" } }, + "node_modules/@img/colour": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", + "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18" + } + }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.2.tgz", - "integrity": "sha512-OfXHZPppddivUJnqyKoi5YVeHRkkNE2zUFT2gbpKxp/JZCFYEYubnMg+gOp6lWfasPrTS+KPosKqdI+ELYVDtg==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", "cpu": [ "arm64" ], @@ -410,13 +420,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.1.0" + "@img/sharp-libvips-darwin-arm64": "1.2.4" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.2.tgz", - "integrity": "sha512-dYvWqmjU9VxqXmjEtjmvHnGqF8GrVjM2Epj9rJ6BUIXvk8slvNDJbhGFvIoXzkDhrJC2jUxNLz/GUjjvSzfw+g==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", "cpu": [ "x64" ], @@ -432,13 +442,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.1.0" + "@img/sharp-libvips-darwin-x64": "1.2.4" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.1.0.tgz", - "integrity": "sha512-HZ/JUmPwrJSoM4DIQPv/BfNh9yrOA8tlBbqbLz4JZ5uew2+o22Ik+tHQJcih7QJuSa0zo5coHTfD5J8inqj9DA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", "cpu": [ "arm64" ], @@ -452,9 +462,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.1.0.tgz", - "integrity": "sha512-Xzc2ToEmHN+hfvsl9wja0RlnXEgpKNmftriQp6XzY/RaSfwD9th+MSh0WQKzUreLKKINb3afirxW7A0fz2YWuQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", "cpu": [ "x64" ], @@ -468,9 +478,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.1.0.tgz", - "integrity": "sha512-s8BAd0lwUIvYCJyRdFqvsj+BJIpDBSxs6ivrOPm/R7piTs5UIwY5OjXrP2bqXC9/moGsyRa37eYWYCOGVXxVrA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", "cpu": [ "arm" ], @@ -484,9 +494,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.1.0.tgz", - "integrity": "sha512-IVfGJa7gjChDET1dK9SekxFFdflarnUB8PwW8aGwEoF3oAsSDuNUTYS+SKDOyOJxQyDC1aPFMuRYLoDInyV9Ew==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", "cpu": [ "arm64" ], @@ -500,9 +510,9 @@ } }, "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.1.0.tgz", - "integrity": "sha512-tiXxFZFbhnkWE2LA8oQj7KYR+bWBkiV2nilRldT7bqoEZ4HiDOcePr9wVDAZPi/Id5fT1oY9iGnDq20cwUz8lQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", "cpu": [ "ppc64" ], @@ -515,10 +525,26 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.1.0.tgz", - "integrity": "sha512-xukSwvhguw7COyzvmjydRb3x/09+21HykyapcZchiCUkTThEQEOMtBj9UhkaBRLuBrgLFzQ2wbxdeCCJW/jgJA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", "cpu": [ "s390x" ], @@ -532,9 +558,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.1.0.tgz", - "integrity": "sha512-yRj2+reB8iMg9W5sULM3S74jVS7zqSzHG3Ol/twnAAkAhnGQnpjj6e4ayUz7V+FpKypwgs82xbRdYtchTTUB+Q==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", "cpu": [ "x64" ], @@ -548,9 +574,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.1.0.tgz", - "integrity": "sha512-jYZdG+whg0MDK+q2COKbYidaqW/WTz0cc1E+tMAusiDygrM4ypmSCjOJPmFTvHHJ8j/6cAGyeDWZOsK06tP33w==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", "cpu": [ "arm64" ], @@ -564,9 +590,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.1.0.tgz", - "integrity": "sha512-wK7SBdwrAiycjXdkPnGCPLjYb9lD4l6Ze2gSdAGVZrEL05AOUJESWU2lhlC+Ffn5/G+VKuSm6zzbQSzFX/P65A==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", "cpu": [ "x64" ], @@ -580,9 +606,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.2.tgz", - "integrity": "sha512-0DZzkvuEOqQUP9mo2kjjKNok5AmnOr1jB2XYjkaoNRwpAYMDzRmAqUIa1nRi58S2WswqSfPOWLNOr0FDT3H5RQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", "cpu": [ "arm" ], @@ -598,13 +624,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.1.0" + "@img/sharp-libvips-linux-arm": "1.2.4" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.2.tgz", - "integrity": "sha512-D8n8wgWmPDakc83LORcfJepdOSN6MvWNzzz2ux0MnIbOqdieRZwVYY32zxVx+IFUT8er5KPcyU3XXsn+GzG/0Q==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", "cpu": [ "arm64" ], @@ -620,13 +646,57 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.1.0" + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.2.tgz", - "integrity": "sha512-EGZ1xwhBI7dNISwxjChqBGELCWMGDvmxZXKjQRuqMrakhO8QoMgqCrdjnAqJq/CScxfRn+Bb7suXBElKQpPDiw==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", "cpu": [ "s390x" ], @@ -642,13 +712,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.1.0" + "@img/sharp-libvips-linux-s390x": "1.2.4" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.2.tgz", - "integrity": "sha512-sD7J+h5nFLMMmOXYH4DD9UtSNBD05tWSSdWAcEyzqW8Cn5UxXvsHAxmxSesYUsTOBmUnjtxghKDl15EvfqLFbQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", "cpu": [ "x64" ], @@ -664,13 +734,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.1.0" + "@img/sharp-libvips-linux-x64": "1.2.4" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.2.tgz", - "integrity": "sha512-NEE2vQ6wcxYav1/A22OOxoSOGiKnNmDzCYFOZ949xFmrWZOVII1Bp3NqVVpvj+3UeHMFyN5eP/V5hzViQ5CZNA==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", "cpu": [ "arm64" ], @@ -686,13 +756,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.1.0" + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.2.tgz", - "integrity": "sha512-DOYMrDm5E6/8bm/yQLCWyuDJwUnlevR8xtF8bs+gjZ7cyUNYXiSf/E8Kp0Ss5xasIaXSHzb888V1BE4i1hFhAA==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", "cpu": [ "x64" ], @@ -708,20 +778,20 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.1.0" + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.2.tgz", - "integrity": "sha512-/VI4mdlJ9zkaq53MbIG6rZY+QRN3MLbR6usYlgITEzi4Rpx5S6LFKsycOQjkOGmqTNmkIdLjEvooFKwww6OpdQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", "cpu": [ "wasm32" ], "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { - "@emnapi/runtime": "^1.4.3" + "@emnapi/runtime": "^1.7.0" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0" @@ -731,9 +801,9 @@ } }, "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.2.tgz", - "integrity": "sha512-cfP/r9FdS63VA5k0xiqaNaEoGxBg9k7uE+RQGzuK9fHt7jib4zAVVseR9LsE4gJcNWgT6APKMNnCcnyOtmSEUQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", "cpu": [ "arm64" ], @@ -750,9 +820,9 @@ } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.2.tgz", - "integrity": "sha512-QLjGGvAbj0X/FXl8n1WbtQ6iVBpWU7JO94u/P2M4a8CFYsvQi4GW2mRy/JqkRx0qpBzaOdKJKw8uc930EX2AHw==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", "cpu": [ "ia32" ], @@ -769,9 +839,9 @@ } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.2.tgz", - "integrity": "sha512-aUdT6zEYtDKCaxkofmmJDJYGCf0+pJg3eU9/oBuqvEeoB9dKI6ZLc/1iLJCTuJQDO4ptntAlkUmHgGjyuobZbw==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", "cpu": [ "x64" ], @@ -1258,15 +1328,15 @@ } }, "node_modules/@next/env": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/@next/env/-/env-15.3.3.tgz", - "integrity": "sha512-OdiMrzCl2Xi0VTjiQQUK0Xh7bJHnOuET2s+3V+Y40WJBAXrJeGA3f+I8MZJ/YQ3mVGi5XGR1L66oFlgqXhQ4Vw==", + "version": "15.5.9", + "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.9.tgz", + "integrity": "sha512-4GlTZ+EJM7WaW2HEZcyU317tIQDjkQIyENDLxYJfSWlfqguN+dHkZgyQTV/7ykvobU7yEH5gKvreNrH4B6QgIg==", "license": "MIT" }, "node_modules/@next/swc-darwin-arm64": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.3.3.tgz", - "integrity": "sha512-WRJERLuH+O3oYB4yZNVahSVFmtxRNjNF1I1c34tYMoJb0Pve+7/RaLAJJizyYiFhjYNGHRAE1Ri2Fd23zgDqhg==", + "version": "15.5.7", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.7.tgz", + "integrity": "sha512-IZwtxCEpI91HVU/rAUOOobWSZv4P2DeTtNaCdHqLcTJU4wdNXgAySvKa/qJCgR5m6KI8UsKDXtO2B31jcaw1Yw==", "cpu": [ "arm64" ], @@ -1280,9 +1350,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.3.3.tgz", - "integrity": "sha512-XHdzH/yBc55lu78k/XwtuFR/ZXUTcflpRXcsu0nKmF45U96jt1tsOZhVrn5YH+paw66zOANpOnFQ9i6/j+UYvw==", + "version": "15.5.7", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.7.tgz", + "integrity": "sha512-UP6CaDBcqaCBuiq/gfCEJw7sPEoX1aIjZHnBWN9v9qYHQdMKvCKcAVs4OX1vIjeE+tC5EIuwDTVIoXpUes29lg==", "cpu": [ "x64" ], @@ -1296,9 +1366,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.3.3.tgz", - "integrity": "sha512-VZ3sYL2LXB8znNGcjhocikEkag/8xiLgnvQts41tq6i+wql63SMS1Q6N8RVXHw5pEUjiof+II3HkDd7GFcgkzw==", + "version": "15.5.7", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.7.tgz", + "integrity": "sha512-NCslw3GrNIw7OgmRBxHtdWFQYhexoUCq+0oS2ccjyYLtcn1SzGzeM54jpTFonIMUjNbHmpKpziXnpxhSWLcmBA==", "cpu": [ "arm64" ], @@ -1312,9 +1382,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.3.3.tgz", - "integrity": "sha512-h6Y1fLU4RWAp1HPNJWDYBQ+e3G7sLckyBXhmH9ajn8l/RSMnhbuPBV/fXmy3muMcVwoJdHL+UtzRzs0nXOf9SA==", + "version": "15.5.7", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.7.tgz", + "integrity": "sha512-nfymt+SE5cvtTrG9u1wdoxBr9bVB7mtKTcj0ltRn6gkP/2Nu1zM5ei8rwP9qKQP0Y//umK+TtkKgNtfboBxRrw==", "cpu": [ "arm64" ], @@ -1328,9 +1398,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.3.3.tgz", - "integrity": "sha512-jJ8HRiF3N8Zw6hGlytCj5BiHyG/K+fnTKVDEKvUCyiQ/0r5tgwO7OgaRiOjjRoIx2vwLR+Rz8hQoPrnmFbJdfw==", + "version": "15.5.7", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.7.tgz", + "integrity": "sha512-hvXcZvCaaEbCZcVzcY7E1uXN9xWZfFvkNHwbe/n4OkRhFWrs1J1QV+4U1BN06tXLdaS4DazEGXwgqnu/VMcmqw==", "cpu": [ "x64" ], @@ -1344,9 +1414,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.3.3.tgz", - "integrity": "sha512-HrUcTr4N+RgiiGn3jjeT6Oo208UT/7BuTr7K0mdKRBtTbT4v9zJqCDKO97DUqqoBK1qyzP1RwvrWTvU6EPh/Cw==", + "version": "15.5.7", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.7.tgz", + "integrity": "sha512-4IUO539b8FmF0odY6/SqANJdgwn1xs1GkPO5doZugwZ3ETF6JUdckk7RGmsfSf7ws8Qb2YB5It33mvNL/0acqA==", "cpu": [ "x64" ], @@ -1360,9 +1430,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.3.3.tgz", - "integrity": "sha512-SxorONgi6K7ZUysMtRF3mIeHC5aA3IQLmKFQzU0OuhuUYwpOBc1ypaLJLP5Bf3M9k53KUUUj4vTPwzGvl/NwlQ==", + "version": "15.5.7", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.7.tgz", + "integrity": "sha512-CpJVTkYI3ZajQkC5vajM7/ApKJUOlm6uP4BknM3XKvJ7VXAvCqSjSLmM0LKdYzn6nBJVSjdclx8nYJSa3xlTgQ==", "cpu": [ "arm64" ], @@ -1376,9 +1446,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.3.3.tgz", - "integrity": "sha512-4QZG6F8enl9/S2+yIiOiju0iCTFd93d8VC1q9LZS4p/Xuk81W2QDjCFeoogmrWWkAD59z8ZxepBQap2dKS5ruw==", + "version": "15.5.7", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.7.tgz", + "integrity": "sha512-gMzgBX164I6DN+9/PGA+9dQiwmTkE4TloBNx8Kv9UiGARsr9Nba7IpcBRA1iTV9vwlYnrE3Uy6I7Aj6qLjQuqw==", "cpu": [ "x64" ], @@ -1491,668 +1561,6 @@ "node": ">=8" } }, - "node_modules/@parcel/core": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/core/-/core-2.15.2.tgz", - "integrity": "sha512-yIFtxeLPLbTkpNuXGmnBX1U51unxv+gRoH/I5IcyD/vRL2Kp/cQU6YJWTSGK5sWG1Fgo+1Z2DeYp914Yd4a1WQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "@mischnic/json-sourcemap": "^0.1.1", - "@parcel/cache": "2.15.2", - "@parcel/diagnostic": "2.15.2", - "@parcel/events": "2.15.2", - "@parcel/feature-flags": "2.15.2", - "@parcel/fs": "2.15.2", - "@parcel/graph": "3.5.2", - "@parcel/logger": "2.15.2", - "@parcel/package-manager": "2.15.2", - "@parcel/plugin": "2.15.2", - "@parcel/profiler": "2.15.2", - "@parcel/rust": "2.15.2", - "@parcel/source-map": "^2.1.1", - "@parcel/types": "2.15.2", - "@parcel/utils": "2.15.2", - "@parcel/workers": "2.15.2", - "base-x": "^3.0.11", - "browserslist": "^4.24.5", - "clone": "^2.1.2", - "dotenv": "^16.5.0", - "dotenv-expand": "^11.0.7", - "json5": "^2.2.3", - "msgpackr": "^1.11.2", - "nullthrows": "^1.1.1", - "semver": "^7.7.1" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/cache": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/cache/-/cache-2.15.2.tgz", - "integrity": "sha512-xYVNKWUHT5hCxo+9nBy9xm7NVfk/jswo+SrU12pXtJm4S5kyK7/PaNkiXxnDu/Hiec2s9BqG/7ny5WBX+i/fAw==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/fs": "2.15.2", - "@parcel/logger": "2.15.2", - "@parcel/utils": "2.15.2", - "lmdb": "2.8.5" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "peerDependencies": { - "@parcel/core": "^2.15.2" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/codeframe": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/codeframe/-/codeframe-2.15.2.tgz", - "integrity": "sha512-uzcHUXBXV+vUqXE7SR6Et60GauPGTWvc381pVzCzc90VQJyWY/xyRRIgcA+4MLi2+lQj+w4Uq9H9qg+hMx/JFg==", - "license": "MIT", - "peer": true, - "dependencies": { - "chalk": "^4.1.2" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/diagnostic": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/diagnostic/-/diagnostic-2.15.2.tgz", - "integrity": "sha512-lsIF59BgfLzN3SP5VM42pa9lilcotEoF42H2RgnqLe3KACcNcbbtvjyjlvac+iaSRix4gEkuZa6376X6p7DkFQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "@mischnic/json-sourcemap": "^0.1.1", - "nullthrows": "^1.1.1" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/events": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/events/-/events-2.15.2.tgz", - "integrity": "sha512-CxXVuYz/K3sDIquM+3Pemxhppb8Q/mRayxqxZtXHoKbhiLBeyX+pLz2v9Hr0R7fiN6naV00IG48Zc5aArHXR4w==", - "license": "MIT", - "peer": true, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/fs": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/fs/-/fs-2.15.2.tgz", - "integrity": "sha512-/Xe+eFbxH43vBCZD+L0nkyIKo8i/nYQpRqzum4YTEoG8WHdcwNl12L9dOcM6EwpaCf6amNVjzBQJMwQ+6E1Y4A==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/feature-flags": "2.15.2", - "@parcel/rust": "2.15.2", - "@parcel/types-internal": "2.15.2", - "@parcel/utils": "2.15.2", - "@parcel/watcher": "^2.0.7", - "@parcel/workers": "2.15.2" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "peerDependencies": { - "@parcel/core": "^2.15.2" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/logger": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/logger/-/logger-2.15.2.tgz", - "integrity": "sha512-naF3dXcvO1lZvtCi6kCTaXhB1cqRwWkRifQRfEei+yp0QZqZF9dmWwZzMOefst/PTl3RaW014vrwFtiegdqsbQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/diagnostic": "2.15.2", - "@parcel/events": "2.15.2" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/markdown-ansi": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/markdown-ansi/-/markdown-ansi-2.15.2.tgz", - "integrity": "sha512-qioxe3Gw/khhrZXeF3tmJeChoq70prxGqVhJylsnGimxHbxjLo3i8Jo8Thi36GiGcOTYSeyF/2tMo9BW2t2vqA==", - "license": "MIT", - "peer": true, - "dependencies": { - "chalk": "^4.1.2" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/node-resolver-core": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/@parcel/node-resolver-core/-/node-resolver-core-3.6.2.tgz", - "integrity": "sha512-MOWpFAuKnVMSZSoXZ9OG1Z7BNSW9IVnDA3DM3c8UYrSR8My7Wng0aen0MyjC3s98N1FEwCodESGfu0+7PpZOIA==", - "license": "MIT", - "peer": true, - "dependencies": { - "@mischnic/json-sourcemap": "^0.1.1", - "@parcel/diagnostic": "2.15.2", - "@parcel/fs": "2.15.2", - "@parcel/rust": "2.15.2", - "@parcel/utils": "2.15.2", - "nullthrows": "^1.1.1", - "semver": "^7.7.1" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/package-manager": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/package-manager/-/package-manager-2.15.2.tgz", - "integrity": "sha512-0n8QupNyXp9CJZV6LohBpAqopLecQrave4kHG/T9CeCeqlJcQnYs+N+zio4mPlv7jXpnJHy+CF96Ce2wy/n1+Q==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/diagnostic": "2.15.2", - "@parcel/fs": "2.15.2", - "@parcel/logger": "2.15.2", - "@parcel/node-resolver-core": "3.6.2", - "@parcel/types": "2.15.2", - "@parcel/utils": "2.15.2", - "@parcel/workers": "2.15.2", - "@swc/core": "^1.11.24", - "semver": "^7.7.1" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "peerDependencies": { - "@parcel/core": "^2.15.2" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/plugin": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/plugin/-/plugin-2.15.2.tgz", - "integrity": "sha512-5ii1OpD/lGdpvy5AS1jChpCwEZP0eFaucy8szOjmfl4oZIeaHRHbZ5R0/3O1Hy8tY1IJF87HUKd+XV0iyD48zA==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/types": "2.15.2" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/profiler": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/profiler/-/profiler-2.15.2.tgz", - "integrity": "sha512-hLTI6TIRr/tGgjTbsCqW4Avl2x8FMAHLDlDhNYjivX6ccfZmilEJnIcdKr2QtdgcaSulfRLTd5bt6uJWJ2ecKg==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/diagnostic": "2.15.2", - "@parcel/events": "2.15.2", - "@parcel/types-internal": "2.15.2", - "chrome-trace-event": "^1.0.2" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/rust": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/rust/-/rust-2.15.2.tgz", - "integrity": "sha512-6ZIVsSnkwxvDDVaxiYK4bWtVaJBYaFQuRvcxfCMQHEzFpWl9mdZVbCs3+g69Ere7a3e2sk87B41d/FIhoaz5xw==", - "license": "MIT", - "peer": true, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "optionalDependencies": { - "@parcel/rust-darwin-arm64": "2.15.2", - "@parcel/rust-darwin-x64": "2.15.2", - "@parcel/rust-linux-arm-gnueabihf": "2.15.2", - "@parcel/rust-linux-arm64-gnu": "2.15.2", - "@parcel/rust-linux-arm64-musl": "2.15.2", - "@parcel/rust-linux-x64-gnu": "2.15.2", - "@parcel/rust-linux-x64-musl": "2.15.2", - "@parcel/rust-win32-x64-msvc": "2.15.2" - }, - "peerDependencies": { - "napi-wasm": "^1.1.2" - }, - "peerDependenciesMeta": { - "napi-wasm": { - "optional": true - } - } - }, - "node_modules/@parcel/core/node_modules/@parcel/types": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/types/-/types-2.15.2.tgz", - "integrity": "sha512-APVvBVVG8RIMLN5hERa2POkPkEtrNUqRbQlKpoNYlIYZaYxKzb9+4MH4cVkmkGfYk3FGU3K5RnxSxMMWsu4tdw==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/types-internal": "2.15.2", - "@parcel/workers": "2.15.2" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/utils": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/utils/-/utils-2.15.2.tgz", - "integrity": "sha512-SQ77yZyeLZf5Teq5aMAViuXKoN7JRnYZ7Pdere1FD8ZuS7E34THA4jjJKxKu9Bqtezgm+gpN1gMbSKMBfbmIZA==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/codeframe": "2.15.2", - "@parcel/diagnostic": "2.15.2", - "@parcel/logger": "2.15.2", - "@parcel/markdown-ansi": "2.15.2", - "@parcel/rust": "2.15.2", - "@parcel/source-map": "^2.1.1", - "chalk": "^4.1.2", - "nullthrows": "^1.1.1" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/core/node_modules/@parcel/workers": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/workers/-/workers-2.15.2.tgz", - "integrity": "sha512-uQWM3Zzkk+vzFYrLQvU/oeM1LC6/EDPvpdgtvdwkUqYC6O1Oei+9cWz6Uv5UDCwizeJKt+3PyE2rB9idbEkmsQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/diagnostic": "2.15.2", - "@parcel/logger": "2.15.2", - "@parcel/profiler": "2.15.2", - "@parcel/types-internal": "2.15.2", - "@parcel/utils": "2.15.2", - "nullthrows": "^1.1.1" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "peerDependencies": { - "@parcel/core": "^2.15.2" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.11.29.tgz", - "integrity": "sha512-g4mThMIpWbNhV8G2rWp5a5/Igv8/2UFRJx2yImrLGMgrDDYZIopqZ/z0jZxDgqNA1QDx93rpwNF7jGsxVWcMlA==", - "hasInstallScript": true, - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@swc/counter": "^0.1.3", - "@swc/types": "^0.1.21" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/swc" - }, - "optionalDependencies": { - "@swc/core-darwin-arm64": "1.11.29", - "@swc/core-darwin-x64": "1.11.29", - "@swc/core-linux-arm-gnueabihf": "1.11.29", - "@swc/core-linux-arm64-gnu": "1.11.29", - "@swc/core-linux-arm64-musl": "1.11.29", - "@swc/core-linux-x64-gnu": "1.11.29", - "@swc/core-linux-x64-musl": "1.11.29", - "@swc/core-win32-arm64-msvc": "1.11.29", - "@swc/core-win32-ia32-msvc": "1.11.29", - "@swc/core-win32-x64-msvc": "1.11.29" - }, - "peerDependencies": { - "@swc/helpers": ">=0.5.17" - }, - "peerDependenciesMeta": { - "@swc/helpers": { - "optional": true - } - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-darwin-arm64": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.11.29.tgz", - "integrity": "sha512-whsCX7URzbuS5aET58c75Dloby3Gtj/ITk2vc4WW6pSDQKSPDuONsIcZ7B2ng8oz0K6ttbi4p3H/PNPQLJ4maQ==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-darwin-x64": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.11.29.tgz", - "integrity": "sha512-S3eTo/KYFk+76cWJRgX30hylN5XkSmjYtCBnM4jPLYn7L6zWYEPajsFLmruQEiTEDUg0gBEWLMNyUeghtswouw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-linux-arm-gnueabihf": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.11.29.tgz", - "integrity": "sha512-o9gdshbzkUMG6azldHdmKklcfrcMx+a23d/2qHQHPDLUPAN+Trd+sDQUYArK5Fcm7TlpG4sczz95ghN0DMkM7g==", - "cpu": [ - "arm" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-linux-arm64-gnu": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.11.29.tgz", - "integrity": "sha512-sLoaciOgUKQF1KX9T6hPGzvhOQaJn+3DHy4LOHeXhQqvBgr+7QcZ+hl4uixPKTzxk6hy6Hb0QOvQEdBAAR1gXw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-linux-arm64-musl": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.11.29.tgz", - "integrity": "sha512-PwjB10BC0N+Ce7RU/L23eYch6lXFHz7r3NFavIcwDNa/AAqywfxyxh13OeRy+P0cg7NDpWEETWspXeI4Ek8otw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-linux-x64-gnu": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.11.29.tgz", - "integrity": "sha512-i62vBVoPaVe9A3mc6gJG07n0/e7FVeAvdD9uzZTtGLiuIfVfIBta8EMquzvf+POLycSk79Z6lRhGPZPJPYiQaA==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-linux-x64-musl": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.11.29.tgz", - "integrity": "sha512-YER0XU1xqFdK0hKkfSVX1YIyCvMDI7K07GIpefPvcfyNGs38AXKhb2byySDjbVxkdl4dycaxxhRyhQ2gKSlsFQ==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-win32-arm64-msvc": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.11.29.tgz", - "integrity": "sha512-po+WHw+k9g6FAg5IJ+sMwtA/fIUL3zPQ4m/uJgONBATCVnDDkyW6dBA49uHNVtSEvjvhuD8DVWdFP847YTcITw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-win32-ia32-msvc": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.11.29.tgz", - "integrity": "sha512-h+NjOrbqdRBYr5ItmStmQt6x3tnhqgwbj9YxdGPepbTDamFv7vFnhZR0YfB3jz3UKJ8H3uGJ65Zw1VsC+xpFkg==", - "cpu": [ - "ia32" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/core-win32-x64-msvc": { - "version": "1.11.29", - "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.11.29.tgz", - "integrity": "sha512-Q8cs2BDV9wqDvqobkXOYdC+pLUSEpX/KvI0Dgfun1F+LzuLotRFuDhrvkU9ETJA6OnD2+Fn/ieHgloiKA/Mn/g==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/@swc/helpers": { - "version": "0.5.17", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz", - "integrity": "sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==", - "license": "Apache-2.0", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.8.0" - } - }, - "node_modules/@parcel/core/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "peer": true, - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@parcel/core/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "peer": true, - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/@parcel/core/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/@parcel/core/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT", - "peer": true - }, - "node_modules/@parcel/core/node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "license": "MIT", - "peer": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/@parcel/core/node_modules/semver": { - "version": "7.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", - "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", - "license": "ISC", - "peer": true, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@parcel/core/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "license": "MIT", - "peer": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/@parcel/diagnostic": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/@parcel/diagnostic/-/diagnostic-2.12.0.tgz", @@ -2181,20 +1589,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/@parcel/feature-flags": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/feature-flags/-/feature-flags-2.15.2.tgz", - "integrity": "sha512-6oiuLd3ypk4GY8X9/l/GrngzSddHW8yF8DrYA++TkaPDtTz4llanza/p7RIk/ltdV3hmBxnH4vjWtciJEcbQww==", - "license": "MIT", - "peer": true, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, "node_modules/@parcel/fs": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/@parcel/fs/-/fs-2.12.0.tgz", @@ -2217,24 +1611,6 @@ "@parcel/core": "^2.12.0" } }, - "node_modules/@parcel/graph": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@parcel/graph/-/graph-3.5.2.tgz", - "integrity": "sha512-SsKKRPotNALU5R5r5WOsP+6FsuaNkk9L0Bmu1UzeyyrHiQPO1OVBYCsX+NtsGDAdDX7oOkGqgfkavJHrAG/BFA==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/feature-flags": "2.15.2", - "nullthrows": "^1.1.1" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, "node_modules/@parcel/logger": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/@parcel/logger/-/logger-2.12.0.tgz", @@ -2501,174 +1877,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/@parcel/rust-darwin-arm64": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/rust-darwin-arm64/-/rust-darwin-arm64-2.15.2.tgz", - "integrity": "sha512-IK5mo/7bNym1ODMWD92D2URGcAq2K/9BasRlfjWI/Gh74l3lH4EFadUfgM88L+MVCV3WTg8ht5ZA0Iyp+IQ1JQ==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/rust-darwin-x64": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/rust-darwin-x64/-/rust-darwin-x64-2.15.2.tgz", - "integrity": "sha512-J30ukJXCzXsYNlYvYsaPEAEzfCZGXVIkXtPSVpWPwcaReqFUyT2bm4I8DHoeas0JwMNaeNlJhksaJA/iomqlwA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/rust-linux-arm-gnueabihf": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/rust-linux-arm-gnueabihf/-/rust-linux-arm-gnueabihf-2.15.2.tgz", - "integrity": "sha512-WpPddkviw8IkRRnT/dRyD3Uzvy6Yuoy5vvtDmpnrR2bJnEz5uQI3TlhMtQo7R+j6aIrDsGFJKBeo9Z0ga0ebNQ==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/rust-linux-arm64-gnu": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/rust-linux-arm64-gnu/-/rust-linux-arm64-gnu-2.15.2.tgz", - "integrity": "sha512-RzD7Gw0QqyUoWaVrtCU+v5J5pg6bybVNknqlEY4jfcJDgJHsM1V91DwJwxnI4ikG/uMedl0I40dl59x/Vo01Ow==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/rust-linux-arm64-musl": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/rust-linux-arm64-musl/-/rust-linux-arm64-musl-2.15.2.tgz", - "integrity": "sha512-mWoL7kCITrEOO0GQ+LqGUylX+6b3nsV60Lzrz2N0Pgzz3EbGS0d4gDKkjxpi6BoR+h4KL7nLhj4hhbm0OHIc4A==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/rust-linux-x64-gnu": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/rust-linux-x64-gnu/-/rust-linux-x64-gnu-2.15.2.tgz", - "integrity": "sha512-aI8bKZTEZNYmgURiAfrgpmaoEArnMRvosfsOKnGykTjmHgsBxO/CGguFj5a4wlAZTVWcTGfs4krnUKtF9Hw6Rw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/rust-linux-x64-musl": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/rust-linux-x64-musl/-/rust-linux-x64-musl-2.15.2.tgz", - "integrity": "sha512-FpQOraPTjGfbHipjdbYpQLlMIRDoVL+Kl9ak+6mt0SbvP3QaXGosQXyhw0ZoNszqVLjIwC0OHEjAHdtcO6ZUvQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/rust-win32-x64-msvc": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/rust-win32-x64-msvc/-/rust-win32-x64-msvc-2.15.2.tgz", - "integrity": "sha512-aSXkPc+KYAT6MnYgw2urXuDvipPkD90uJBKtSn3MY+fGOfzEluK7j0F5NdH88oTzrGVhRQxnxfe3Fc+IRhsaFQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, "node_modules/@parcel/source-map": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@parcel/source-map/-/source-map-2.1.1.tgz", @@ -2752,37 +1960,6 @@ "utility-types": "^3.10.0" } }, - "node_modules/@parcel/types-internal": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/types-internal/-/types-internal-2.15.2.tgz", - "integrity": "sha512-nmMpYeG4le49nvr8FsJYGEwhCZxcrm89tvkX8xGod1yXcShEZNWVVY9ezZLKxMrVMdBveqNUW8IZCij5iFDqdQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "@parcel/diagnostic": "2.15.2", - "@parcel/feature-flags": "2.15.2", - "@parcel/source-map": "^2.1.1", - "utility-types": "^3.11.0" - } - }, - "node_modules/@parcel/types-internal/node_modules/@parcel/diagnostic": { - "version": "2.15.2", - "resolved": "https://registry.npmjs.org/@parcel/diagnostic/-/diagnostic-2.15.2.tgz", - "integrity": "sha512-lsIF59BgfLzN3SP5VM42pa9lilcotEoF42H2RgnqLe3KACcNcbbtvjyjlvac+iaSRix4gEkuZa6376X6p7DkFQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "@mischnic/json-sourcemap": "^0.1.1", - "nullthrows": "^1.1.1" - }, - "engines": { - "node": ">= 16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, "node_modules/@parcel/utils": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/@parcel/utils/-/utils-2.12.0.tgz", @@ -3952,16 +3129,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/base-x": { - "version": "3.0.11", - "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.11.tgz", - "integrity": "sha512-xz7wQ8xDhdyP7tQxwdteLYeFfS68tSMNCZ/Y37WJ4bhGfKPpqEIlmIyueQHqOyoPhE6xNUqjzRr8ra0eF9VRvA==", - "license": "MIT", - "peer": true, - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -4010,17 +3177,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/busboy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", - "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", - "dependencies": { - "streamsearch": "^1.1.0" - }, - "engines": { - "node": ">=10.16.0" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -4178,16 +3334,6 @@ "node": ">=4" } }, - "node_modules/clone": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", - "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", - "license": "MIT", - "peer": true, - "engines": { - "node": ">=0.8" - } - }, "node_modules/clsx": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.0.tgz", @@ -4196,20 +3342,6 @@ "node": ">=6" } }, - "node_modules/color": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", - "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "license": "MIT", - "optional": true, - "dependencies": { - "color-convert": "^2.0.1", - "color-string": "^1.9.0" - }, - "engines": { - "node": ">=12.5.0" - } - }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -4223,37 +3355,6 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "license": "MIT", - "optional": true, - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/color/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT", - "optional": true - }, "node_modules/comma-separated-tokens": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", @@ -4962,35 +4063,6 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, - "node_modules/dotenv": { - "version": "16.5.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", - "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", - "license": "BSD-2-Clause", - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/dotenv-expand": { - "version": "11.0.7", - "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.7.tgz", - "integrity": "sha512-zIHwmZPRshsCdpMDyVsqGmgyP0yT8GAgXUnkdAoJisxvf33k7yO6OuoKmcTGuXPWSsm8Oh88nZicRLA9Y0rUeA==", - "license": "BSD-2-Clause", - "peer": true, - "dependencies": { - "dotenv": "^16.4.5" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, "node_modules/electron-to-chromium": { "version": "1.5.158", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.158.tgz", @@ -7570,15 +6642,13 @@ } }, "node_modules/next": { - "version": "15.3.3", - "resolved": "https://registry.npmjs.org/next/-/next-15.3.3.tgz", - "integrity": "sha512-JqNj29hHNmCLtNvd090SyRbXJiivQ+58XjCcrC50Crb5g5u2zi7Y2YivbsEfzk6AtVI80akdOQbaMZwWB1Hthw==", + "version": "15.5.9", + "resolved": "https://registry.npmjs.org/next/-/next-15.5.9.tgz", + "integrity": "sha512-agNLK89seZEtC5zUHwtut0+tNrc0Xw4FT/Dg+B/VLEo9pAcS9rtTKpek3V6kVcVwsB2YlqMaHdfZL4eLEVYuCg==", "license": "MIT", "dependencies": { - "@next/env": "15.3.3", - "@swc/counter": "0.1.3", + "@next/env": "15.5.9", "@swc/helpers": "0.5.15", - "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" @@ -7590,19 +6660,19 @@ "node": "^18.18.0 || ^19.8.0 || >= 20.0.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "15.3.3", - "@next/swc-darwin-x64": "15.3.3", - "@next/swc-linux-arm64-gnu": "15.3.3", - "@next/swc-linux-arm64-musl": "15.3.3", - "@next/swc-linux-x64-gnu": "15.3.3", - "@next/swc-linux-x64-musl": "15.3.3", - "@next/swc-win32-arm64-msvc": "15.3.3", - "@next/swc-win32-x64-msvc": "15.3.3", - "sharp": "^0.34.1" + "@next/swc-darwin-arm64": "15.5.7", + "@next/swc-darwin-x64": "15.5.7", + "@next/swc-linux-arm64-gnu": "15.5.7", + "@next/swc-linux-arm64-musl": "15.5.7", + "@next/swc-linux-x64-gnu": "15.5.7", + "@next/swc-linux-x64-musl": "15.5.7", + "@next/swc-win32-arm64-msvc": "15.5.7", + "@next/swc-win32-x64-msvc": "15.5.7", + "sharp": "^0.34.3" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", - "@playwright/test": "^1.41.2", + "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", @@ -7968,6 +7038,8 @@ }, "node_modules/npm/node_modules/@isaacs/cliui": { "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", "inBundle": true, "license": "ISC", "dependencies": { @@ -7984,6 +7056,8 @@ }, "node_modules/npm/node_modules/@isaacs/cliui/node_modules/ansi-regex": { "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", "inBundle": true, "license": "MIT", "engines": { @@ -7995,6 +7069,8 @@ }, "node_modules/npm/node_modules/@isaacs/cliui/node_modules/emoji-regex": { "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "inBundle": true, "license": "MIT" }, @@ -8016,6 +7092,8 @@ }, "node_modules/npm/node_modules/@isaacs/cliui/node_modules/strip-ansi": { "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", "inBundle": true, "license": "MIT", "dependencies": { @@ -8289,6 +7367,8 @@ }, "node_modules/npm/node_modules/@pkgjs/parseargs": { "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", "inBundle": true, "license": "MIT", "optional": true, @@ -8415,6 +7495,8 @@ }, "node_modules/npm/node_modules/ansi-regex": { "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "inBundle": true, "license": "MIT", "engines": { @@ -8423,6 +7505,8 @@ }, "node_modules/npm/node_modules/ansi-styles": { "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", "inBundle": true, "license": "MIT", "engines": { @@ -8452,6 +7536,8 @@ }, "node_modules/npm/node_modules/balanced-match": { "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "inBundle": true, "license": "MIT" }, @@ -8482,6 +7568,8 @@ }, "node_modules/npm/node_modules/brace-expansion": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "inBundle": true, "license": "MIT", "dependencies": { @@ -8614,6 +7702,8 @@ }, "node_modules/npm/node_modules/color-convert": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "inBundle": true, "license": "MIT", "dependencies": { @@ -8625,6 +7715,8 @@ }, "node_modules/npm/node_modules/color-name": { "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "inBundle": true, "license": "MIT" }, @@ -8660,6 +7752,8 @@ }, "node_modules/npm/node_modules/cross-spawn": { "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "inBundle": true, "license": "MIT", "dependencies": { @@ -8687,6 +7781,8 @@ }, "node_modules/npm/node_modules/cssesc": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", "inBundle": true, "license": "MIT", "bin": { @@ -8714,6 +7810,8 @@ }, "node_modules/npm/node_modules/debug/node_modules/ms": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "inBundle": true, "license": "MIT" }, @@ -8738,11 +7836,15 @@ }, "node_modules/npm/node_modules/eastasianwidth": { "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", "inBundle": true, "license": "MIT" }, "node_modules/npm/node_modules/emoji-regex": { "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "inBundle": true, "license": "MIT" }, @@ -8783,6 +7885,8 @@ }, "node_modules/npm/node_modules/foreground-child": { "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", "inBundle": true, "license": "ISC", "dependencies": { @@ -8809,6 +7913,8 @@ }, "node_modules/npm/node_modules/function-bind": { "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", "inBundle": true, "license": "MIT", "funding": { @@ -8856,6 +7962,8 @@ }, "node_modules/npm/node_modules/graceful-fs": { "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "inBundle": true, "license": "ISC" }, @@ -9031,6 +8139,8 @@ }, "node_modules/npm/node_modules/is-fullwidth-code-point": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "inBundle": true, "license": "MIT", "engines": { @@ -9044,6 +8154,8 @@ }, "node_modules/npm/node_modules/isexe": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "inBundle": true, "license": "ISC" }, @@ -9305,6 +8417,8 @@ }, "node_modules/npm/node_modules/minipass": { "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", "inBundle": true, "license": "ISC", "engines": { @@ -9719,6 +8833,8 @@ }, "node_modules/npm/node_modules/path-key": { "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "inBundle": true, "license": "MIT", "engines": { @@ -9866,6 +8982,8 @@ }, "node_modules/npm/node_modules/safer-buffer": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "inBundle": true, "license": "MIT", "optional": true @@ -9886,6 +9004,8 @@ }, "node_modules/npm/node_modules/semver/node_modules/lru-cache": { "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "inBundle": true, "license": "ISC", "dependencies": { @@ -9902,6 +9022,8 @@ }, "node_modules/npm/node_modules/shebang-command": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "inBundle": true, "license": "MIT", "dependencies": { @@ -9913,6 +9035,8 @@ }, "node_modules/npm/node_modules/shebang-regex": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "inBundle": true, "license": "MIT", "engines": { @@ -9921,6 +9045,8 @@ }, "node_modules/npm/node_modules/signal-exit": { "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", "inBundle": true, "license": "ISC", "engines": { @@ -10022,6 +9148,8 @@ }, "node_modules/npm/node_modules/string-width": { "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "inBundle": true, "license": "MIT", "dependencies": { @@ -10036,6 +9164,8 @@ "node_modules/npm/node_modules/string-width-cjs": { "name": "string-width", "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "inBundle": true, "license": "MIT", "dependencies": { @@ -10049,6 +9179,8 @@ }, "node_modules/npm/node_modules/strip-ansi": { "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "inBundle": true, "license": "MIT", "dependencies": { @@ -10061,6 +9193,8 @@ "node_modules/npm/node_modules/strip-ansi-cjs": { "name": "strip-ansi", "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "inBundle": true, "license": "MIT", "dependencies": { @@ -10182,6 +9316,8 @@ }, "node_modules/npm/node_modules/util-deprecate": { "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "inBundle": true, "license": "MIT" }, @@ -10250,6 +9386,8 @@ }, "node_modules/npm/node_modules/wrap-ansi": { "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", "inBundle": true, "license": "MIT", "dependencies": { @@ -10267,6 +9405,8 @@ "node_modules/npm/node_modules/wrap-ansi-cjs": { "name": "wrap-ansi", "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "inBundle": true, "license": "MIT", "dependencies": { @@ -10297,6 +9437,8 @@ }, "node_modules/npm/node_modules/wrap-ansi/node_modules/ansi-regex": { "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", "inBundle": true, "license": "MIT", "engines": { @@ -10308,6 +9450,8 @@ }, "node_modules/npm/node_modules/wrap-ansi/node_modules/emoji-regex": { "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "inBundle": true, "license": "MIT" }, @@ -10329,6 +9473,8 @@ }, "node_modules/npm/node_modules/wrap-ansi/node_modules/strip-ansi": { "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", "inBundle": true, "license": "MIT", "dependencies": { @@ -10355,6 +9501,8 @@ }, "node_modules/npm/node_modules/yallist": { "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "inBundle": true, "license": "ISC" }, @@ -10920,27 +10068,6 @@ "node": ">=6" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "peer": true - }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -10993,16 +10120,16 @@ "integrity": "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==" }, "node_modules/sharp": { - "version": "0.34.2", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.2.tgz", - "integrity": "sha512-lszvBmB9QURERtyKT2bNmsgxXK0ShJrL/fvqlonCo7e6xBF8nT8xU6pW+PMIbLsz0RxQk3rgH9kd8UmvOzlMJg==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", "hasInstallScript": true, "license": "Apache-2.0", "optional": true, "dependencies": { - "color": "^4.2.3", - "detect-libc": "^2.0.4", - "semver": "^7.7.2" + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0" @@ -11011,33 +10138,36 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.2", - "@img/sharp-darwin-x64": "0.34.2", - "@img/sharp-libvips-darwin-arm64": "1.1.0", - "@img/sharp-libvips-darwin-x64": "1.1.0", - "@img/sharp-libvips-linux-arm": "1.1.0", - "@img/sharp-libvips-linux-arm64": "1.1.0", - "@img/sharp-libvips-linux-ppc64": "1.1.0", - "@img/sharp-libvips-linux-s390x": "1.1.0", - "@img/sharp-libvips-linux-x64": "1.1.0", - "@img/sharp-libvips-linuxmusl-arm64": "1.1.0", - "@img/sharp-libvips-linuxmusl-x64": "1.1.0", - "@img/sharp-linux-arm": "0.34.2", - "@img/sharp-linux-arm64": "0.34.2", - "@img/sharp-linux-s390x": "0.34.2", - "@img/sharp-linux-x64": "0.34.2", - "@img/sharp-linuxmusl-arm64": "0.34.2", - "@img/sharp-linuxmusl-x64": "0.34.2", - "@img/sharp-wasm32": "0.34.2", - "@img/sharp-win32-arm64": "0.34.2", - "@img/sharp-win32-ia32": "0.34.2", - "@img/sharp-win32-x64": "0.34.2" + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" } }, "node_modules/sharp/node_modules/detect-libc": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", - "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", "license": "Apache-2.0", "optional": true, "engines": { @@ -11045,9 +10175,9 @@ } }, "node_modules/sharp/node_modules/semver": { - "version": "7.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", - "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", "license": "ISC", "optional": true, "bin": { @@ -11068,23 +10198,6 @@ "vscode-textmate": "^8.0.0" } }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "license": "MIT", - "optional": true, - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/simple-swizzle/node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", - "license": "MIT", - "optional": true - }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -11154,14 +10267,6 @@ "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==", "deprecated": "Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility" }, - "node_modules/streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/stringify-entities": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.3.tgz", @@ -11361,20 +10466,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/typescript": { - "version": "5.8.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", - "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", - "license": "Apache-2.0", - "peer": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, "node_modules/uc.micro": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", diff --git a/docs/package.json b/docs/package.json index f17a7b01..32719002 100644 --- a/docs/package.json +++ b/docs/package.json @@ -8,7 +8,7 @@ "dependencies": { "@vercel/analytics": "^1.1.1", "docsgpt-react": "^0.5.1", - "next": "^15.3.3", + "next": "^15.5.9", "nextra": "^2.13.2", "nextra-theme-docs": "^2.13.2", "react": "^18.2.0", From 2a4ab3aca13c0e7f324e69f52959c9d9df3ccd6c Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 17 Dec 2025 14:07:14 +0000 Subject: [PATCH 22/93] Fix history leftover (#2198) * fix: history leftover * fix: unbound result --- application/worker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/application/worker.py b/application/worker.py index 8442e7f4..d0c4bec9 100755 --- a/application/worker.py +++ b/application/worker.py @@ -960,13 +960,17 @@ def agent_webhook_worker(self, agent_id, payload): result = run_agent_logic(agent_config, input_data) except Exception as e: logging.error(f"Error running agent logic: {e}", exc_info=True) - return {"status": "error", "error": str(e)} - finally: self.update_state(state="PROGRESS", meta={"current": 100}) logging.info( f"Webhook processed for agent {agent_id}", extra={"agent_id": agent_id} ) - return {"status": "success", "result": result} + return {"status": "error"} + + self.update_state(state="PROGRESS", meta={"current": 100}) + logging.info( + f"Webhook processed for agent {agent_id}", extra={"agent_id": agent_id} + ) + return {"status": "success", "result": result} def ingest_connector( From b0d4576a95b5f1098838d890a55d5e65e59c8973 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 18 Dec 2025 13:27:40 +0000 Subject: [PATCH 23/93] fix: improve error handling in agent webhook worker --- application/worker.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/application/worker.py b/application/worker.py index d0c4bec9..f45e94a5 100755 --- a/application/worker.py +++ b/application/worker.py @@ -960,17 +960,14 @@ def agent_webhook_worker(self, agent_id, payload): result = run_agent_logic(agent_config, input_data) except Exception as e: logging.error(f"Error running agent logic: {e}", exc_info=True) - self.update_state(state="PROGRESS", meta={"current": 100}) + return {"status": "error"} + else: logging.info( f"Webhook processed for agent {agent_id}", extra={"agent_id": agent_id} ) - return {"status": "error"} - - self.update_state(state="PROGRESS", meta={"current": 100}) - logging.info( - f"Webhook processed for agent {agent_id}", extra={"agent_id": agent_id} - ) - return {"status": "success", "result": result} + return {"status": "success", "result": result} + finally: + self.update_state(state="PROGRESS", meta={"current": 100}) def ingest_connector( From a69a0e100f44ef9401001353702fb9e8c57e4e1e Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 19 Dec 2025 00:17:59 +0000 Subject: [PATCH 24/93] fix: update dependencies in requirements.txt (#2201) --- application/requirements.txt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/application/requirements.txt b/application/requirements.txt index c02f75c0..82763f89 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -1,9 +1,10 @@ anthropic==0.75.0 -boto3==1.42.6 +boto3==1.42.12 beautifulsoup4==4.14.3 celery==5.6.0 cryptography==46.0.3 dataclasses-json==0.6.7 +docling>=2.16.0 docx2txt==0.8 duckduckgo-search==8.1.1 ebooklib==0.20 @@ -17,7 +18,7 @@ fastmcp==2.13.3 flask-restx==1.3.2 google-genai==1.54.0 google-api-python-client==2.187.0 -google-auth-httplib2==0.2.0 +google-auth-httplib2==0.3.0 google-auth-oauthlib==1.2.2 gTTS==2.5.4 gunicorn==23.0.0 @@ -45,17 +46,17 @@ mypy-extensions==1.0.0 networkx==3.6.1 numpy==2.2.1 openai==2.9.0 -openapi3-parser==1.1.21 +openapi3-parser==1.1.22 orjson==3.11.5 packaging==24.2 pandas==2.2.3 openpyxl==3.1.5 pathable==0.4.4 -pillow==12.0.0 +pillow portalocker>=2.7.0,<3.0.0 -prance==23.6.21.0 +prance==25.4.8.0 prompt-toolkit==3.0.51 -protobuf==5.29.3 +protobuf==6.33.2 psycopg2-binary==2.9.11 py==1.11.0 pydantic @@ -75,7 +76,7 @@ retry==0.9.2 sentence-transformers==5.1.2 tiktoken==0.12.0 tokenizers==0.22.1 -torch==2.7.0 +torch==2.9.1 tqdm==4.67.1 transformers==4.57.3 typing-extensions==4.15.0 From d90b1c57e5acd4caffe6e0e9dde10f77e9346091 Mon Sep 17 00:00:00 2001 From: JustACodeA Date: Fri, 19 Dec 2025 17:25:29 +0100 Subject: [PATCH 25/93] feat: add hover animation to conversation context menu button (#2168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add hover animation to conversation context menu button Adds visual feedback when hovering over the three-dot menu button in conversation tiles. This makes it clear that the submenu is being targeted rather than the parent item. Changes: - Added rounded hover background with smooth transition - Increased clickable area for better UX - Supports both light and dark themes Closes #2097 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude * Update frontend/src/conversation/ConversationTile.tsx Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Claude Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- frontend/src/conversation/ConversationTile.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/conversation/ConversationTile.tsx b/frontend/src/conversation/ConversationTile.tsx index b18b59fc..fe1a8fa1 100644 --- a/frontend/src/conversation/ConversationTile.tsx +++ b/frontend/src/conversation/ConversationTile.tsx @@ -248,7 +248,7 @@ export default function ConversationTile({ event.stopPropagation(); setOpen(!isOpen); }} - className="mr-2 flex w-4 justify-center" + className="mr-2 flex h-6 w-6 items-center justify-center rounded-full transition-colors duration-200 hover:bg-gray-200 dark:hover:bg-gray-700" > menu From 3ad38f53fdddfd10bca8b0f506d1c33c085f0a2c Mon Sep 17 00:00:00 2001 From: JustACodeA Date: Fri, 19 Dec 2025 17:29:58 +0100 Subject: [PATCH 26/93] fix: update Node.js version to 22 for Vite compatibility (#2169) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the frontend Dockerfile from Node 20.6.1 to Node 22 to resolve compatibility issues with Vite dependencies. Closes #2157 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude From a6fafa6a4d2ebebf16a5262b93cb6bb814731d0c Mon Sep 17 00:00:00 2001 From: Rahul Badade Date: Fri, 19 Dec 2025 22:27:57 +0530 Subject: [PATCH 27/93] Fix: Autoselect input text box on pageload and conversation reset (#2177) (#2194) * Fix: Autoselect input text box on pageload and conversation reset - Added autoFocus to useEffect dependency array in MessageInput - Added key prop to MessageInput to force remount on conversation reset - Implemented refocus after message submission - Removed duplicate input clearing logic in handleKeyDown Fixes #2177 * fix: optimize input handling --------- Co-authored-by: Alex --- frontend/src/components/MessageInput.tsx | 28 ++++++++++++++++------ frontend/src/conversation/Conversation.tsx | 1 + 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/MessageInput.tsx b/frontend/src/components/MessageInput.tsx index 750b0b12..86bb0f6d 100644 --- a/frontend/src/components/MessageInput.tsx +++ b/frontend/src/components/MessageInput.tsx @@ -500,7 +500,7 @@ export default function MessageInput({ return () => clearInterval(interval); }, [attachments, dispatch]); - const handleInput = () => { + const handleInput = useCallback(() => { if (inputRef.current) { if (window.innerWidth < 350) inputRef.current.style.height = 'auto'; else inputRef.current.style.height = '64px'; @@ -509,12 +509,21 @@ export default function MessageInput({ 96, )}px`; } - }; + }, []); + + const isMountedRef = useRef(true); + + useEffect(() => { + isMountedRef.current = true; + return () => { + isMountedRef.current = false; + }; + }, []); useEffect(() => { if (autoFocus) inputRef.current?.focus(); handleInput(); - }, []); + }, [autoFocus, handleInput]); const handleChange = (e: React.ChangeEvent) => { setValue(e.target.value); @@ -525,10 +534,7 @@ export default function MessageInput({ if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); handleSubmit(); - if (inputRef.current) { - inputRef.current.value = ''; - handleInput(); - } + handleInput(); } }; @@ -565,6 +571,14 @@ export default function MessageInput({ if (value.trim() && !loading) { onSubmit(value); setValue(''); + // Refocus input after submission if autoFocus is enabled + if (autoFocus) { + setTimeout(() => { + if (isMountedRef.current) { + inputRef.current?.focus(); + } + }, 0); + } } }; diff --git a/frontend/src/conversation/Conversation.tsx b/frontend/src/conversation/Conversation.tsx index 2ab193ec..62fe7c54 100644 --- a/frontend/src/conversation/Conversation.tsx +++ b/frontend/src/conversation/Conversation.tsx @@ -179,6 +179,7 @@ export default function Conversation() {
{ handleQuestionSubmission(text); }} From 7958d29e132d44e6218b820a1ef200f7f5de70d4 Mon Sep 17 00:00:00 2001 From: Yash <118480441+ItsYash1421@users.noreply.github.com> Date: Fri, 19 Dec 2025 22:38:56 +0530 Subject: [PATCH 28/93] Fix: Import external-link.svg properly in AgentDetailsModal (#2191) --- frontend/src/modals/AgentDetailsModal.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/modals/AgentDetailsModal.tsx b/frontend/src/modals/AgentDetailsModal.tsx index b2ae6826..25f32833 100644 --- a/frontend/src/modals/AgentDetailsModal.tsx +++ b/frontend/src/modals/AgentDetailsModal.tsx @@ -1,7 +1,7 @@ import { useEffect, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; - +import ExternalLinkIcon from '../assets/external-link.svg'; import { Agent } from '../agents/types'; import userService from '../api/services/userService'; import CopyButton from '../components/CopyButton'; @@ -123,7 +123,7 @@ export default function AgentDetailsModal({ {t('modals.agentDetails.learnMore')} External link @@ -168,7 +168,7 @@ export default function AgentDetailsModal({ > {t('modals.agentDetails.test')} External link @@ -210,7 +210,7 @@ export default function AgentDetailsModal({ {t('modals.agentDetails.learnMore')} External link From 40c3e5568c79bf2754d7aca8b9352fa8c52d63a3 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 21 Dec 2025 22:51:06 +0000 Subject: [PATCH 29/93] fix search (#2210) * fix search * fix ruff --- application/api/answer/__init__.py | 2 + application/api/answer/routes/search.py | 186 ++++++++ application/requirements.txt | 8 +- tests/api/answer/routes/test_search.py | 561 ++++++++++++++++++++++++ 4 files changed, 753 insertions(+), 4 deletions(-) create mode 100644 application/api/answer/routes/search.py create mode 100644 tests/api/answer/routes/test_search.py diff --git a/application/api/answer/__init__.py b/application/api/answer/__init__.py index 861c922d..a10b9b5f 100644 --- a/application/api/answer/__init__.py +++ b/application/api/answer/__init__.py @@ -3,6 +3,7 @@ from flask import Blueprint from application.api import api from application.api.answer.routes.answer import AnswerResource from application.api.answer.routes.base import answer_ns +from application.api.answer.routes.search import SearchResource from application.api.answer.routes.stream import StreamResource @@ -14,6 +15,7 @@ api.add_namespace(answer_ns) def init_answer_routes(): api.add_resource(StreamResource, "/stream") api.add_resource(AnswerResource, "/api/answer") + api.add_resource(SearchResource, "/api/search") init_answer_routes() diff --git a/application/api/answer/routes/search.py b/application/api/answer/routes/search.py new file mode 100644 index 00000000..16ebdb82 --- /dev/null +++ b/application/api/answer/routes/search.py @@ -0,0 +1,186 @@ +import logging +from typing import Any, Dict, List + +from flask import make_response, request +from flask_restx import fields, Resource + +from bson.dbref import DBRef + +from application.api.answer.routes.base import answer_ns +from application.core.mongo_db import MongoDB +from application.core.settings import settings +from application.vectorstore.vector_creator import VectorCreator + +logger = logging.getLogger(__name__) + + +@answer_ns.route("/api/search") +class SearchResource(Resource): + """Fast search endpoint for retrieving relevant documents""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + mongo = MongoDB.get_client() + self.db = mongo[settings.MONGO_DB_NAME] + self.agents_collection = self.db["agents"] + + search_model = answer_ns.model( + "SearchModel", + { + "question": fields.String( + required=True, description="Search query" + ), + "api_key": fields.String( + required=True, description="API key for authentication" + ), + "chunks": fields.Integer( + required=False, default=5, description="Number of results to return" + ), + }, + ) + + def _get_sources_from_api_key(self, api_key: str) -> List[str]: + """Get source IDs connected to the API key/agent. + + """ + agent_data = self.agents_collection.find_one({"key": api_key}) + if not agent_data: + return [] + + source_ids = [] + + # Handle multiple sources (only if non-empty) + sources = agent_data.get("sources", []) + if sources and isinstance(sources, list) and len(sources) > 0: + for source_ref in sources: + # Skip "default" - it's a placeholder, not an actual vectorstore + if source_ref == "default": + continue + elif isinstance(source_ref, DBRef): + source_doc = self.db.dereference(source_ref) + if source_doc: + source_ids.append(str(source_doc["_id"])) + + # Handle single source (legacy) - check if sources was empty or didn't yield results + if not source_ids: + source = agent_data.get("source") + if isinstance(source, DBRef): + source_doc = self.db.dereference(source) + if source_doc: + source_ids.append(str(source_doc["_id"])) + # Skip "default" - it's a placeholder, not an actual vectorstore + elif source and source != "default": + source_ids.append(source) + + return source_ids + + def _search_vectorstores( + self, query: str, source_ids: List[str], chunks: int + ) -> List[Dict[str, Any]]: + """Search across vectorstores and return results""" + if not source_ids: + return [] + + results = [] + chunks_per_source = max(1, chunks // len(source_ids)) + seen_texts = set() + + for source_id in source_ids: + if not source_id or not source_id.strip(): + continue + + try: + docsearch = VectorCreator.create_vectorstore( + settings.VECTOR_STORE, source_id, settings.EMBEDDINGS_KEY + ) + docs = docsearch.search(query, k=chunks_per_source * 2) + + for doc in docs: + if len(results) >= chunks: + break + + if hasattr(doc, "page_content") and hasattr(doc, "metadata"): + page_content = doc.page_content + metadata = doc.metadata + else: + page_content = doc.get("text", doc.get("page_content", "")) + metadata = doc.get("metadata", {}) + + # Skip duplicates + text_hash = hash(page_content[:200]) + if text_hash in seen_texts: + continue + seen_texts.add(text_hash) + + title = metadata.get( + "title", metadata.get("post_title", "") + ) + if not isinstance(title, str): + title = str(title) if title else "" + + # Clean up title + if title: + title = title.split("/")[-1] + else: + # Use filename or first part of content as title + title = metadata.get("filename", page_content[:50] + "...") + + source = metadata.get("source", source_id) + + results.append({ + "text": page_content, + "title": title, + "source": source, + }) + + if len(results) >= chunks: + break + + except Exception as e: + logger.error( + f"Error searching vectorstore {source_id}: {e}", + exc_info=True, + ) + continue + + return results[:chunks] + + @answer_ns.expect(search_model) + @answer_ns.doc(description="Search for relevant documents based on query") + def post(self): + data = request.get_json() + + question = data.get("question") + api_key = data.get("api_key") + chunks = data.get("chunks", 5) + + if not question: + return make_response({"error": "question is required"}, 400) + + if not api_key: + return make_response({"error": "api_key is required"}, 400) + + # Validate API key + agent = self.agents_collection.find_one({"key": api_key}) + if not agent: + return make_response({"error": "Invalid API key"}, 401) + + try: + # Get sources connected to this API key + source_ids = self._get_sources_from_api_key(api_key) + + if not source_ids: + return make_response([], 200) + + # Perform search + results = self._search_vectorstores(question, source_ids, chunks) + + return make_response(results, 200) + + except Exception as e: + logger.error( + f"/api/search - error: {str(e)}", + extra={"error": str(e)}, + exc_info=True, + ) + return make_response({"error": "Search failed"}, 500) diff --git a/application/requirements.txt b/application/requirements.txt index 82763f89..a6528ca3 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -32,11 +32,11 @@ jsonpointer==3.0.0 kombu==5.6.1 langchain==1.1.3 langchain-community==0.4.1 -langchain-core==1.1.3 +langchain-core==1.2.4 langchain-openai==1.1.1 langchain-text-splitters==1.0.0 langsmith==0.4.58 -lazy-object-proxy==1.10.0 +lazy-object-proxy==1.12.0 lxml==6.0.2 markupsafe==3.0.2 marshmallow==3.26.1 @@ -55,7 +55,7 @@ pathable==0.4.4 pillow portalocker>=2.7.0,<3.0.0 prance==25.4.8.0 -prompt-toolkit==3.0.51 +prompt-toolkit==3.0.52 protobuf==6.33.2 psycopg2-binary==2.9.11 py==1.11.0 @@ -84,7 +84,7 @@ typing-inspect==0.9.0 tzdata==2025.2 urllib3==2.6.1 vine==5.1.0 -wcwidth==0.2.13 +wcwidth==0.2.14 werkzeug>=3.1.0 yarl==1.22.0 markdownify==1.2.2 diff --git a/tests/api/answer/routes/test_search.py b/tests/api/answer/routes/test_search.py new file mode 100644 index 00000000..b397cf3c --- /dev/null +++ b/tests/api/answer/routes/test_search.py @@ -0,0 +1,561 @@ +from unittest.mock import MagicMock, patch + +import pytest +from bson import ObjectId +from bson.dbref import DBRef + + +@pytest.mark.unit +class TestSearchResourceValidation: + def test_returns_error_when_question_missing(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + with flask_app.test_request_context( + json={"api_key": "test_key"} + ): + resource = SearchResource() + result = resource.post() + + assert result.status_code == 400 + assert "question" in result.json["error"] + + def test_returns_error_when_api_key_missing(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + with flask_app.test_request_context( + json={"question": "test query"} + ): + resource = SearchResource() + result = resource.post() + + assert result.status_code == 400 + assert "api_key" in result.json["error"] + + def test_returns_error_for_invalid_api_key(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + with flask_app.test_request_context( + json={"question": "test query", "api_key": "invalid_key"} + ): + resource = SearchResource() + result = resource.post() + + assert result.status_code == 401 + assert "Invalid API key" in result.json["error"] + + +@pytest.mark.unit +class TestGetSourcesFromApiKey: + def test_returns_empty_list_when_agent_not_found(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + result = resource._get_sources_from_api_key("nonexistent_key") + + assert result == [] + + def test_returns_source_id_from_dbref(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + source_id = ObjectId() + agent_id = ObjectId() + + sources_collection = mock_mongo_db[settings.MONGO_DB_NAME]["sources"] + sources_collection.insert_one( + {"_id": source_id, "name": "Test Source"} + ) + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "source": DBRef("sources", source_id), + "sources": [], + } + ) + + resource = SearchResource() + result = resource._get_sources_from_api_key("test_api_key") + + assert len(result) == 1 + assert result[0] == str(source_id) + + def test_returns_multiple_sources_from_sources_array( + self, mock_mongo_db, flask_app + ): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + source_id_1 = ObjectId() + source_id_2 = ObjectId() + agent_id = ObjectId() + + sources_collection = mock_mongo_db[settings.MONGO_DB_NAME]["sources"] + sources_collection.insert_one({"_id": source_id_1, "name": "Source 1"}) + sources_collection.insert_one({"_id": source_id_2, "name": "Source 2"}) + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "sources": [ + DBRef("sources", source_id_1), + DBRef("sources", source_id_2), + ], + } + ) + + resource = SearchResource() + result = resource._get_sources_from_api_key("test_api_key") + + assert len(result) == 2 + assert str(source_id_1) in result + assert str(source_id_2) in result + + def test_skips_default_source_in_sources_array(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + source_id = ObjectId() + agent_id = ObjectId() + + sources_collection = mock_mongo_db[settings.MONGO_DB_NAME]["sources"] + sources_collection.insert_one({"_id": source_id, "name": "Test Source"}) + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "sources": ["default", DBRef("sources", source_id)], + } + ) + + resource = SearchResource() + result = resource._get_sources_from_api_key("test_api_key") + + assert len(result) == 1 + assert result[0] == str(source_id) + assert "default" not in result + + def test_skips_default_source_in_legacy_field(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + agent_id = ObjectId() + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "source": "default", + "sources": [], + } + ) + + resource = SearchResource() + result = resource._get_sources_from_api_key("test_api_key") + + assert result == [] + + def test_falls_back_to_legacy_source_when_sources_empty( + self, mock_mongo_db, flask_app + ): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + source_id = ObjectId() + agent_id = ObjectId() + + sources_collection = mock_mongo_db[settings.MONGO_DB_NAME]["sources"] + sources_collection.insert_one({"_id": source_id, "name": "Test Source"}) + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "source": DBRef("sources", source_id), + "sources": [], + } + ) + + resource = SearchResource() + result = resource._get_sources_from_api_key("test_api_key") + + assert len(result) == 1 + assert result[0] == str(source_id) + + def test_handles_string_source_id(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + agent_id = ObjectId() + source_id = "custom_source_id" + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "source": source_id, + "sources": [], + } + ) + + resource = SearchResource() + result = resource._get_sources_from_api_key("test_api_key") + + assert len(result) == 1 + assert result[0] == source_id + + +@pytest.mark.unit +class TestSearchVectorstores: + def test_returns_empty_when_no_source_ids(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + result = resource._search_vectorstores("test query", [], 5) + + assert result == [] + + def test_skips_empty_source_ids(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_vectorstore = MagicMock() + mock_vectorstore.search.return_value = [] + mock_create.return_value = mock_vectorstore + + result = resource._search_vectorstores("test query", ["", " "], 5) + + mock_create.assert_not_called() + assert result == [] + + def test_returns_search_results(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + mock_doc = { + "text": "Test content", + "page_content": "Test content", + "metadata": { + "title": "Test Title", + "source": "/path/to/doc", + }, + } + + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_vectorstore = MagicMock() + mock_vectorstore.search.return_value = [mock_doc] + mock_create.return_value = mock_vectorstore + + result = resource._search_vectorstores("test query", ["source_id"], 5) + + assert len(result) == 1 + assert result[0]["text"] == "Test content" + assert result[0]["title"] == "Test Title" + assert result[0]["source"] == "/path/to/doc" + + def test_handles_langchain_document_format(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + mock_doc = MagicMock() + mock_doc.page_content = "Langchain content" + mock_doc.metadata = {"title": "LC Title", "source": "/lc/path"} + + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_vectorstore = MagicMock() + mock_vectorstore.search.return_value = [mock_doc] + mock_create.return_value = mock_vectorstore + + result = resource._search_vectorstores("test query", ["source_id"], 5) + + assert len(result) == 1 + assert result[0]["text"] == "Langchain content" + assert result[0]["title"] == "LC Title" + + def test_respects_chunks_limit(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + mock_docs = [ + {"text": f"Content {i}", "metadata": {"title": f"Title {i}"}} + for i in range(10) + ] + + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_vectorstore = MagicMock() + mock_vectorstore.search.return_value = mock_docs + mock_create.return_value = mock_vectorstore + + result = resource._search_vectorstores("test query", ["source_id"], 3) + + assert len(result) == 3 + + def test_deduplicates_results(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + duplicate_text = "Duplicate content " * 20 + mock_docs = [ + {"text": duplicate_text, "metadata": {"title": "Title 1"}}, + {"text": duplicate_text, "metadata": {"title": "Title 2"}}, + {"text": "Unique content", "metadata": {"title": "Title 3"}}, + ] + + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_vectorstore = MagicMock() + mock_vectorstore.search.return_value = mock_docs + mock_create.return_value = mock_vectorstore + + result = resource._search_vectorstores("test query", ["source_id"], 5) + + assert len(result) == 2 + + def test_handles_vectorstore_error_gracefully(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_create.side_effect = Exception("Vectorstore error") + + result = resource._search_vectorstores("test query", ["source_id"], 5) + + assert result == [] + + def test_uses_filename_as_title_fallback(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + mock_doc = { + "text": "Content without title", + "metadata": {"filename": "document.pdf"}, + } + + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_vectorstore = MagicMock() + mock_vectorstore.search.return_value = [mock_doc] + mock_create.return_value = mock_vectorstore + + result = resource._search_vectorstores("test query", ["source_id"], 5) + + assert result[0]["title"] == "document.pdf" + + def test_uses_content_snippet_as_title_last_resort(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + + with flask_app.app_context(): + resource = SearchResource() + + mock_doc = { + "text": "Content without any title metadata at all", + "metadata": {}, + } + + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_vectorstore = MagicMock() + mock_vectorstore.search.return_value = [mock_doc] + mock_create.return_value = mock_vectorstore + + result = resource._search_vectorstores("test query", ["source_id"], 5) + + assert "Content without any title" in result[0]["title"] + assert result[0]["title"].endswith("...") + + +@pytest.mark.unit +class TestSearchEndpoint: + def test_returns_empty_array_when_no_sources(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + agent_id = ObjectId() + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "source": "default", + "sources": [], + } + ) + + with flask_app.test_request_context( + json={"question": "test query", "api_key": "test_api_key"} + ): + resource = SearchResource() + result = resource.post() + + assert result.status_code == 200 + assert result.json == [] + + def test_returns_search_results_successfully(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + source_id = ObjectId() + agent_id = ObjectId() + + sources_collection = mock_mongo_db[settings.MONGO_DB_NAME]["sources"] + sources_collection.insert_one({"_id": source_id, "name": "Test Source"}) + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "source": DBRef("sources", source_id), + "sources": [], + } + ) + + mock_doc = { + "text": "Search result content", + "metadata": {"title": "Result Title", "source": "/doc/path"}, + } + + with flask_app.test_request_context( + json={"question": "test query", "api_key": "test_api_key", "chunks": 5} + ): + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_vectorstore = MagicMock() + mock_vectorstore.search.return_value = [mock_doc] + mock_create.return_value = mock_vectorstore + + resource = SearchResource() + result = resource.post() + + assert result.status_code == 200 + assert len(result.json) == 1 + assert result.json[0]["text"] == "Search result content" + assert result.json[0]["title"] == "Result Title" + + def test_uses_default_chunks_value(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + source_id = ObjectId() + agent_id = ObjectId() + + sources_collection = mock_mongo_db[settings.MONGO_DB_NAME]["sources"] + sources_collection.insert_one({"_id": source_id, "name": "Test Source"}) + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "source": DBRef("sources", source_id), + "sources": [], + } + ) + + with flask_app.test_request_context( + json={"question": "test query", "api_key": "test_api_key"} + ): + with patch( + "application.api.answer.routes.search.VectorCreator.create_vectorstore" + ) as mock_create: + mock_vectorstore = MagicMock() + mock_vectorstore.search.return_value = [] + mock_create.return_value = mock_vectorstore + + resource = SearchResource() + resource.post() + + mock_vectorstore.search.assert_called_once() + call_args = mock_vectorstore.search.call_args + assert call_args[1]["k"] == 10 + + def test_handles_internal_error(self, mock_mongo_db, flask_app): + from application.api.answer.routes.search import SearchResource + from application.core.settings import settings + + with flask_app.app_context(): + source_id = ObjectId() + agent_id = ObjectId() + + sources_collection = mock_mongo_db[settings.MONGO_DB_NAME]["sources"] + sources_collection.insert_one({"_id": source_id, "name": "Test Source"}) + + agents_collection = mock_mongo_db[settings.MONGO_DB_NAME]["agents"] + agents_collection.insert_one( + { + "_id": agent_id, + "key": "test_api_key", + "source": DBRef("sources", source_id), + "sources": [], + } + ) + + with flask_app.test_request_context( + json={"question": "test query", "api_key": "test_api_key"} + ): + resource = SearchResource() + + with patch.object( + resource, "_get_sources_from_api_key" + ) as mock_get_sources: + mock_get_sources.side_effect = Exception("Database error") + + result = resource.post() + + assert result.status_code == 500 + assert "Search failed" in result.json["error"] From 87e24ab96ea7adecbf95cae5dbcc1891dcf4acb2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Dec 2025 12:25:45 +0200 Subject: [PATCH 30/93] chore(deps): bump elevenlabs from 2.26.1 to 2.27.0 in /application (#2203) Bumps [elevenlabs](https://github.com/elevenlabs/elevenlabs-python) from 2.26.1 to 2.27.0. - [Release notes](https://github.com/elevenlabs/elevenlabs-python/releases) - [Commits](https://github.com/elevenlabs/elevenlabs-python/compare/v2.26.1...v2.27.0) --- updated-dependencies: - dependency-name: elevenlabs dependency-version: 2.27.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index a6528ca3..c88cabaf 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -11,7 +11,7 @@ ebooklib==0.20 escodegen==1.0.11 esprima==4.0.1 esutils==1.0.1 -elevenlabs==2.26.1 +elevenlabs==2.27.0 Flask==3.1.2 faiss-cpu==1.13.1 fastmcp==2.13.3 From f91846ce2d1411ed621ba997873a37240cfab9aa Mon Sep 17 00:00:00 2001 From: Akash Bhadana <129368922+AkashBhadana@users.noreply.github.com> Date: Mon, 22 Dec 2025 22:23:30 +0530 Subject: [PATCH 31/93] docs: Update VECTOR_STORE comment to include pgvector (#2211) Co-authored-by: root --- application/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/core/settings.py b/application/core/settings.py index f688df9b..0f49d8fe 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -43,7 +43,7 @@ class Settings(BaseSettings): PARSE_PDF_AS_IMAGE: bool = False PARSE_IMAGE_REMOTE: bool = False VECTOR_STORE: str = ( - "faiss" # "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb" + "faiss" # "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb" or "pgvector" ) RETRIEVERS_ENABLED: list = ["classic_rag"] AGENT_NAME: str = "classic" From 5b6cfa6ecc7c69f0a2370e059076c2ced672192f Mon Sep 17 00:00:00 2001 From: Siddhant Rai <47355538+siiddhantt@users.noreply.github.com> Date: Tue, 23 Dec 2025 19:07:44 +0530 Subject: [PATCH 32/93] feat: enhance API tool with body serialization and content type handling (#2192) * feat: enhance API tool with body serialization and content type handling * feat: enhance ToolConfig with import functionality and user action management - Added ImportSpecModal to allow importing actions into the tool configuration. - Implemented search functionality for user actions with expandable action details. - Introduced method colors for better visual distinction of HTTP methods. - Updated APIActionType and ParameterGroupType to include optional 'required' field. - Refactored action rendering to improve usability and maintainability. * feat: add base URL input to ImportSpecModal for action URL customization * feat: update TestBaseAgentTools to include 'required' field for parameters * feat: standardize API call timeout to DEFAULT_TIMEOUT constant * feat: add import specification functionality and related translations for multiple languages --------- Co-authored-by: Alex --- application/agents/base.py | 30 +- .../agents/tools/api_body_serializer.py | 323 +++++ application/agents/tools/api_tool.py | 242 +++- application/agents/tools/spec_parser.py | 342 +++++ application/api/user/tools/routes.py | 55 + frontend/src/api/endpoints.ts | 1 + frontend/src/api/services/userService.ts | 5 + frontend/src/locale/de.json | 21 + frontend/src/locale/en.json | 21 + frontend/src/locale/es.json | 21 + frontend/src/locale/jp.json | 21 + frontend/src/locale/ru.json | 21 + frontend/src/locale/zh-TW.json | 21 + frontend/src/locale/zh.json | 21 + frontend/src/modals/ImportSpecModal.tsx | 321 +++++ frontend/src/settings/ToolConfig.tsx | 1164 ++++++++++++----- frontend/src/settings/types/index.ts | 17 +- tests/agents/test_base_agent.py | 8 +- 18 files changed, 2308 insertions(+), 347 deletions(-) create mode 100644 application/agents/tools/api_body_serializer.py create mode 100644 application/agents/tools/spec_parser.py create mode 100644 frontend/src/modals/ImportSpecModal.tsx diff --git a/application/agents/base.py b/application/agents/base.py index b2d79c03..44df7ee4 100644 --- a/application/agents/base.py +++ b/application/agents/base.py @@ -120,10 +120,10 @@ class BaseAgent(ABC): params["properties"][k] = { key: value for key, value in v.items() - if key != "filled_by_llm" and key != "value" + if key not in ("filled_by_llm", "value", "required") } - - params["required"].append(k) + if v.get("required", False): + params["required"].append(k) return params def _prepare_tools(self, tools_dict): @@ -219,7 +219,11 @@ class BaseAgent(ABC): for param_type, target_dict in param_types.items(): if param_type in action_data and action_data[param_type].get("properties"): for param, details in action_data[param_type]["properties"].items(): - if param not in call_args and "value" in details: + if ( + param not in call_args + and "value" in details + and details["value"] + ): target_dict[param] = details["value"] for param, value in call_args.items(): for param_type, target_dict in param_types.items(): @@ -232,12 +236,20 @@ class BaseAgent(ABC): # Prepare tool_config and add tool_id for memory tools if tool_data["name"] == "api_tool": + action_config = tool_data["config"]["actions"][action_name] tool_config = { - "url": tool_data["config"]["actions"][action_name]["url"], - "method": tool_data["config"]["actions"][action_name]["method"], + "url": action_config["url"], + "method": action_config["method"], "headers": headers, "query_params": query_params, } + if "body_content_type" in action_config: + tool_config["body_content_type"] = action_config.get( + "body_content_type", "application/json" + ) + tool_config["body_encoding_rules"] = action_config.get( + "body_encoding_rules", {} + ) else: tool_config = tool_data["config"].copy() if tool_data["config"] else {} # Add tool_id from MongoDB _id for tools that need instance isolation (like memory tool) @@ -247,15 +259,15 @@ class BaseAgent(ABC): tool = tm.load_tool( tool_data["name"], tool_config=tool_config, - user_id=self.user, # Pass user ID for MCP tools credential decryption + user_id=self.user, ) if tool_data["name"] == "api_tool": - print( + logger.debug( f"Executing api: {action_name} with query_params: {query_params}, headers: {headers}, body: {body}" ) result = tool.execute_action(action_name, **body) else: - print(f"Executing tool: {action_name} with args: {call_args}") + logger.debug(f"Executing tool: {action_name} with args: {call_args}") result = tool.execute_action(action_name, **parameters) tool_call_data["result"] = ( f"{str(result)[:50]}..." if len(str(result)) > 50 else result diff --git a/application/agents/tools/api_body_serializer.py b/application/agents/tools/api_body_serializer.py new file mode 100644 index 00000000..d23d1fcf --- /dev/null +++ b/application/agents/tools/api_body_serializer.py @@ -0,0 +1,323 @@ +import base64 +import json +import logging +from enum import Enum +from typing import Any, Dict, Optional, Union +from urllib.parse import quote, urlencode + +logger = logging.getLogger(__name__) + + +class ContentType(str, Enum): + """Supported content types for request bodies.""" + + JSON = "application/json" + FORM_URLENCODED = "application/x-www-form-urlencoded" + MULTIPART_FORM_DATA = "multipart/form-data" + TEXT_PLAIN = "text/plain" + XML = "application/xml" + OCTET_STREAM = "application/octet-stream" + + +class RequestBodySerializer: + """Serializes request bodies according to content-type and OpenAPI 3.1 spec.""" + + @staticmethod + def serialize( + body_data: Dict[str, Any], + content_type: str = ContentType.JSON, + encoding_rules: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> tuple[Union[str, bytes], Dict[str, str]]: + """ + Serialize body data to appropriate format. + + Args: + body_data: Dictionary of body parameters + content_type: Content-Type header value + encoding_rules: OpenAPI Encoding Object rules per field + + Returns: + Tuple of (serialized_body, updated_headers_dict) + + Raises: + ValueError: If serialization fails + """ + if not body_data: + return None, {} + + try: + content_type_lower = content_type.lower().split(";")[0].strip() + + if content_type_lower == ContentType.JSON: + return RequestBodySerializer._serialize_json(body_data) + + elif content_type_lower == ContentType.FORM_URLENCODED: + return RequestBodySerializer._serialize_form_urlencoded( + body_data, encoding_rules + ) + + elif content_type_lower == ContentType.MULTIPART_FORM_DATA: + return RequestBodySerializer._serialize_multipart_form_data( + body_data, encoding_rules + ) + + elif content_type_lower == ContentType.TEXT_PLAIN: + return RequestBodySerializer._serialize_text_plain(body_data) + + elif content_type_lower == ContentType.XML: + return RequestBodySerializer._serialize_xml(body_data) + + elif content_type_lower == ContentType.OCTET_STREAM: + return RequestBodySerializer._serialize_octet_stream(body_data) + + else: + logger.warning( + f"Unknown content type: {content_type}, treating as JSON" + ) + return RequestBodySerializer._serialize_json(body_data) + + except Exception as e: + logger.error(f"Error serializing body: {str(e)}", exc_info=True) + raise ValueError(f"Failed to serialize request body: {str(e)}") + + @staticmethod + def _serialize_json(body_data: Dict[str, Any]) -> tuple[str, Dict[str, str]]: + """Serialize body as JSON per OpenAPI spec.""" + try: + serialized = json.dumps( + body_data, separators=(",", ":"), ensure_ascii=False + ) + headers = {"Content-Type": ContentType.JSON.value} + return serialized, headers + except (TypeError, ValueError) as e: + raise ValueError(f"Failed to serialize JSON body: {str(e)}") + + @staticmethod + def _serialize_form_urlencoded( + body_data: Dict[str, Any], + encoding_rules: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> tuple[str, Dict[str, str]]: + """Serialize body as application/x-www-form-urlencoded per RFC1866/RFC3986.""" + encoding_rules = encoding_rules or {} + params = [] + + for key, value in body_data.items(): + if value is None: + continue + + rule = encoding_rules.get(key, {}) + style = rule.get("style", "form") + explode = rule.get("explode", style == "form") + content_type = rule.get("contentType", "text/plain") + + serialized_value = RequestBodySerializer._serialize_form_value( + value, style, explode, content_type, key + ) + + if isinstance(serialized_value, list): + for sv in serialized_value: + params.append((key, sv)) + else: + params.append((key, serialized_value)) + + # Use standard urlencode (replaces space with +) + serialized = urlencode(params, safe="") + headers = {"Content-Type": ContentType.FORM_URLENCODED.value} + return serialized, headers + + @staticmethod + def _serialize_form_value( + value: Any, style: str, explode: bool, content_type: str, key: str + ) -> Union[str, list]: + """Serialize individual form value with encoding rules.""" + if isinstance(value, dict): + if content_type == "application/json": + return json.dumps(value, separators=(",", ":")) + elif content_type == "application/xml": + return RequestBodySerializer._dict_to_xml(value) + else: + if style == "deepObject" and explode: + return [ + f"{RequestBodySerializer._percent_encode(str(v))}" + for v in value.values() + ] + elif explode: + return [ + f"{RequestBodySerializer._percent_encode(str(v))}" + for v in value.values() + ] + else: + pairs = [f"{k},{v}" for k, v in value.items()] + return RequestBodySerializer._percent_encode(",".join(pairs)) + + elif isinstance(value, (list, tuple)): + if explode: + return [ + RequestBodySerializer._percent_encode(str(item)) for item in value + ] + else: + return RequestBodySerializer._percent_encode( + ",".join(str(v) for v in value) + ) + + else: + return RequestBodySerializer._percent_encode(str(value)) + + @staticmethod + def _serialize_multipart_form_data( + body_data: Dict[str, Any], + encoding_rules: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> tuple[bytes, Dict[str, str]]: + """ + Serialize body as multipart/form-data per RFC7578. + + Supports file uploads and encoding rules. + """ + import secrets + + encoding_rules = encoding_rules or {} + boundary = f"----DocsGPT{secrets.token_hex(16)}" + parts = [] + + for key, value in body_data.items(): + if value is None: + continue + + rule = encoding_rules.get(key, {}) + content_type = rule.get("contentType", "text/plain") + headers_rule = rule.get("headers", {}) + + part = RequestBodySerializer._create_multipart_part( + key, value, content_type, headers_rule + ) + parts.append(part) + + body_bytes = f"--{boundary}\r\n".encode("utf-8") + body_bytes += f"--{boundary}\r\n".join(parts).encode("utf-8") + body_bytes += f"\r\n--{boundary}--\r\n".encode("utf-8") + + headers = { + "Content-Type": f"multipart/form-data; boundary={boundary}", + } + return body_bytes, headers + + @staticmethod + def _create_multipart_part( + name: str, value: Any, content_type: str, headers_rule: Dict[str, Any] + ) -> str: + """Create a single multipart/form-data part.""" + headers = [ + f'Content-Disposition: form-data; name="{RequestBodySerializer._percent_encode(name)}"' + ] + + if isinstance(value, bytes): + if content_type == "application/octet-stream": + value_encoded = base64.b64encode(value).decode("utf-8") + else: + value_encoded = value.decode("utf-8", errors="replace") + headers.append(f"Content-Type: {content_type}") + headers.append("Content-Transfer-Encoding: base64") + elif isinstance(value, dict): + if content_type == "application/json": + value_encoded = json.dumps(value, separators=(",", ":")) + elif content_type == "application/xml": + value_encoded = RequestBodySerializer._dict_to_xml(value) + else: + value_encoded = str(value) + headers.append(f"Content-Type: {content_type}") + elif isinstance(value, str) and content_type != "text/plain": + try: + if content_type == "application/json": + json.loads(value) + value_encoded = value + elif content_type == "application/xml": + value_encoded = value + else: + value_encoded = str(value) + except json.JSONDecodeError: + value_encoded = str(value) + headers.append(f"Content-Type: {content_type}") + else: + value_encoded = str(value) + if content_type != "text/plain": + headers.append(f"Content-Type: {content_type}") + + part = "\r\n".join(headers) + "\r\n\r\n" + value_encoded + "\r\n" + return part + + @staticmethod + def _serialize_text_plain(body_data: Dict[str, Any]) -> tuple[str, Dict[str, str]]: + """Serialize body as plain text.""" + if len(body_data) == 1: + value = list(body_data.values())[0] + return str(value), {"Content-Type": ContentType.TEXT_PLAIN.value} + else: + text = "\n".join(f"{k}: {v}" for k, v in body_data.items()) + return text, {"Content-Type": ContentType.TEXT_PLAIN.value} + + @staticmethod + def _serialize_xml(body_data: Dict[str, Any]) -> tuple[str, Dict[str, str]]: + """Serialize body as XML.""" + xml_str = RequestBodySerializer._dict_to_xml(body_data) + return xml_str, {"Content-Type": ContentType.XML.value} + + @staticmethod + def _serialize_octet_stream( + body_data: Dict[str, Any], + ) -> tuple[bytes, Dict[str, str]]: + """Serialize body as binary octet stream.""" + if isinstance(body_data, bytes): + return body_data, {"Content-Type": ContentType.OCTET_STREAM.value} + elif isinstance(body_data, str): + return body_data.encode("utf-8"), { + "Content-Type": ContentType.OCTET_STREAM.value + } + else: + serialized = json.dumps(body_data) + return serialized.encode("utf-8"), { + "Content-Type": ContentType.OCTET_STREAM.value + } + + @staticmethod + def _percent_encode(value: str, safe_chars: str = "") -> str: + """ + Percent-encode per RFC3986. + + Args: + value: String to encode + safe_chars: Additional characters to not encode + """ + return quote(value, safe=safe_chars) + + @staticmethod + def _dict_to_xml(data: Dict[str, Any], root_name: str = "root") -> str: + """ + Convert dict to simple XML format. + """ + + def build_xml(obj: Any, name: str) -> str: + if isinstance(obj, dict): + inner = "".join(build_xml(v, k) for k, v in obj.items()) + return f"<{name}>{inner}" + elif isinstance(obj, (list, tuple)): + items = "".join( + build_xml(item, f"{name[:-1] if name.endswith('s') else name}") + for item in obj + ) + return items + else: + return f"<{name}>{RequestBodySerializer._escape_xml(str(obj))}" + + root = build_xml(data, root_name) + return f'{root}' + + @staticmethod + def _escape_xml(value: str) -> str: + """Escape XML special characters.""" + return ( + value.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) diff --git a/application/agents/tools/api_tool.py b/application/agents/tools/api_tool.py index 063313c4..6bd2eb8d 100644 --- a/application/agents/tools/api_tool.py +++ b/application/agents/tools/api_tool.py @@ -1,72 +1,256 @@ import json +import logging +import re +from typing import Any, Dict, Optional +from urllib.parse import urlencode import requests + +from application.agents.tools.api_body_serializer import ( + ContentType, + RequestBodySerializer, +) from application.agents.tools.base import Tool +logger = logging.getLogger(__name__) + +DEFAULT_TIMEOUT = 90 # seconds + class APITool(Tool): """ API Tool - A flexible tool for performing various API actions (e.g., sending messages, retrieving data) via custom user-specified APIs + A flexible tool for performing various API actions (e.g., sending messages, retrieving data) via custom user-specified APIs. """ def __init__(self, config): self.config = config self.url = config.get("url", "") self.method = config.get("method", "GET") - self.headers = config.get("headers", {"Content-Type": "application/json"}) + self.headers = config.get("headers", {}) self.query_params = config.get("query_params", {}) + self.body_content_type = config.get("body_content_type", ContentType.JSON) + self.body_encoding_rules = config.get("body_encoding_rules", {}) def execute_action(self, action_name, **kwargs): + """Execute an API action with the given arguments.""" return self._make_api_call( - self.url, self.method, self.headers, self.query_params, kwargs + self.url, + self.method, + self.headers, + self.query_params, + kwargs, + self.body_content_type, + self.body_encoding_rules, ) - def _make_api_call(self, url, method, headers, query_params, body): - if query_params: - url = f"{url}?{requests.compat.urlencode(query_params)}" - # if isinstance(body, dict): - # body = json.dumps(body) + def _make_api_call( + self, + url: str, + method: str, + headers: Dict[str, str], + query_params: Dict[str, Any], + body: Dict[str, Any], + content_type: str = ContentType.JSON, + encoding_rules: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> Dict[str, Any]: + """ + Make an API call with proper body serialization and error handling. + + Args: + url: API endpoint URL + method: HTTP method (GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS) + headers: Request headers dict + query_params: URL query parameters + body: Request body as dict + content_type: Content-Type for serialization + encoding_rules: OpenAPI encoding rules + + Returns: + Dict with status_code, data, and message + """ + request_url = url + request_headers = headers.copy() if headers else {} + response = None + try: - print(f"Making API call: {method} {url} with body: {body}") - if body == "{}": - body = None - response = requests.request(method, url, headers=headers, data=body) - response.raise_for_status() - content_type = response.headers.get( - "Content-Type", "application/json" - ).lower() - if "application/json" in content_type: + path_params_used = set() + if query_params: + for match in re.finditer(r"\{([^}]+)\}", request_url): + param_name = match.group(1) + if param_name in query_params: + request_url = request_url.replace( + f"{{{param_name}}}", str(query_params[param_name]) + ) + path_params_used.add(param_name) + remaining_params = { + k: v for k, v in query_params.items() if k not in path_params_used + } + if remaining_params: + query_string = urlencode(remaining_params) + separator = "&" if "?" in request_url else "?" + request_url = f"{request_url}{separator}{query_string}" + # Serialize body based on content type + + if body and body != {}: try: - data = response.json() - except json.JSONDecodeError as e: - print(f"Error decoding JSON: {e}. Raw response: {response.text}") + serialized_body, body_headers = RequestBodySerializer.serialize( + body, content_type, encoding_rules + ) + request_headers.update(body_headers) + except ValueError as e: + logger.error(f"Body serialization failed: {str(e)}") return { - "status_code": response.status_code, - "message": f"API call returned invalid JSON. Error: {e}", - "data": response.text, + "status_code": None, + "message": f"Body serialization error: {str(e)}", + "data": None, } - elif "text/" in content_type or "application/xml" in content_type: - data = response.text - elif not response.content: - data = None else: - print(f"Unsupported content type: {content_type}") - data = response.content + serialized_body = None + if "Content-Type" not in request_headers and method not in [ + "GET", + "HEAD", + "DELETE", + ]: + request_headers["Content-Type"] = ContentType.JSON + logger.debug( + f"API Call: {method} {request_url} | Content-Type: {request_headers.get('Content-Type', 'N/A')}" + ) + + if method.upper() == "GET": + response = requests.get( + request_url, headers=request_headers, timeout=DEFAULT_TIMEOUT + ) + elif method.upper() == "POST": + response = requests.post( + request_url, + data=serialized_body, + headers=request_headers, + timeout=DEFAULT_TIMEOUT, + ) + elif method.upper() == "PUT": + response = requests.put( + request_url, + data=serialized_body, + headers=request_headers, + timeout=DEFAULT_TIMEOUT, + ) + elif method.upper() == "DELETE": + response = requests.delete( + request_url, headers=request_headers, timeout=DEFAULT_TIMEOUT + ) + elif method.upper() == "PATCH": + response = requests.patch( + request_url, + data=serialized_body, + headers=request_headers, + timeout=DEFAULT_TIMEOUT, + ) + elif method.upper() == "HEAD": + response = requests.head( + request_url, headers=request_headers, timeout=DEFAULT_TIMEOUT + ) + elif method.upper() == "OPTIONS": + response = requests.options( + request_url, headers=request_headers, timeout=DEFAULT_TIMEOUT + ) + else: + return { + "status_code": None, + "message": f"Unsupported HTTP method: {method}", + "data": None, + } + response.raise_for_status() + + data = self._parse_response(response) return { "status_code": response.status_code, "data": data, "message": "API call successful.", } + except requests.exceptions.Timeout: + logger.error(f"Request timeout for {request_url}") + return { + "status_code": None, + "message": f"Request timeout ({DEFAULT_TIMEOUT}s exceeded)", + "data": None, + } + except requests.exceptions.ConnectionError as e: + logger.error(f"Connection error: {str(e)}") + return { + "status_code": None, + "message": f"Connection error: {str(e)}", + "data": None, + } + except requests.exceptions.HTTPError as e: + logger.error(f"HTTP error {response.status_code}: {str(e)}") + try: + error_data = response.json() + except (json.JSONDecodeError, ValueError): + error_data = response.text + return { + "status_code": response.status_code, + "message": f"HTTP Error {response.status_code}", + "data": error_data, + } except requests.exceptions.RequestException as e: + logger.error(f"Request failed: {str(e)}") return { "status_code": response.status_code if response else None, "message": f"API call failed: {str(e)}", + "data": None, + } + except Exception as e: + logger.error(f"Unexpected error in API call: {str(e)}", exc_info=True) + return { + "status_code": None, + "message": f"Unexpected error: {str(e)}", + "data": None, } + def _parse_response(self, response: requests.Response) -> Any: + """ + Parse response based on Content-Type header. + + Supports: JSON, XML, plain text, binary data. + """ + content_type = response.headers.get("Content-Type", "").lower() + + if not response.content: + return None + # JSON response + + if "application/json" in content_type: + try: + return response.json() + except json.JSONDecodeError as e: + logger.warning(f"Failed to parse JSON response: {str(e)}") + return response.text + # XML response + + elif "application/xml" in content_type or "text/xml" in content_type: + return response.text + # Plain text response + + elif "text/plain" in content_type or "text/html" in content_type: + return response.text + # Binary/unknown response + + else: + # Try to decode as text first, fall back to base64 + + try: + return response.text + except (UnicodeDecodeError, AttributeError): + import base64 + + return base64.b64encode(response.content).decode("utf-8") + def get_actions_metadata(self): + """Return metadata for available actions (none for API Tool - actions are user-defined).""" return [] def get_config_requirements(self): + """Return configuration requirements for the tool.""" return {} diff --git a/application/agents/tools/spec_parser.py b/application/agents/tools/spec_parser.py new file mode 100644 index 00000000..336f00f8 --- /dev/null +++ b/application/agents/tools/spec_parser.py @@ -0,0 +1,342 @@ +""" +API Specification Parser + +Parses OpenAPI 3.x and Swagger 2.0 specifications and converts them +to API Tool action definitions for use in DocsGPT. +""" + +import json +import logging +import re +from typing import Any, Dict, List, Optional, Tuple + +import yaml + +logger = logging.getLogger(__name__) + +SUPPORTED_METHODS = frozenset( + {"get", "post", "put", "delete", "patch", "head", "options"} +) + + +def parse_spec(spec_content: str) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: + """ + Parse an API specification and convert operations to action definitions. + + Supports OpenAPI 3.x and Swagger 2.0 formats in JSON or YAML. + + Args: + spec_content: Raw specification content as string + + Returns: + Tuple of (metadata dict, list of action dicts) + + Raises: + ValueError: If the spec is invalid or uses an unsupported format + """ + spec = _load_spec(spec_content) + _validate_spec(spec) + + is_swagger = "swagger" in spec + metadata = _extract_metadata(spec, is_swagger) + actions = _extract_actions(spec, is_swagger) + + return metadata, actions + + +def _load_spec(content: str) -> Dict[str, Any]: + """Parse spec content from JSON or YAML string.""" + content = content.strip() + if not content: + raise ValueError("Empty specification content") + try: + if content.startswith("{"): + return json.loads(content) + return yaml.safe_load(content) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON format: {e.msg}") + except yaml.YAMLError as e: + raise ValueError(f"Invalid YAML format: {e}") + + +def _validate_spec(spec: Dict[str, Any]) -> None: + """Validate spec version and required fields.""" + if not isinstance(spec, dict): + raise ValueError("Specification must be a valid object") + openapi_version = spec.get("openapi", "") + swagger_version = spec.get("swagger", "") + + if not (openapi_version.startswith("3.") or swagger_version == "2.0"): + raise ValueError( + "Unsupported specification version. Expected OpenAPI 3.x or Swagger 2.0" + ) + if "paths" not in spec or not spec["paths"]: + raise ValueError("No API paths defined in the specification") + + +def _extract_metadata(spec: Dict[str, Any], is_swagger: bool) -> Dict[str, Any]: + """Extract API metadata from specification.""" + info = spec.get("info", {}) + base_url = _get_base_url(spec, is_swagger) + + return { + "title": info.get("title", "Untitled API"), + "description": (info.get("description", "") or "")[:500], + "version": info.get("version", ""), + "base_url": base_url, + } + + +def _get_base_url(spec: Dict[str, Any], is_swagger: bool) -> str: + """Extract base URL from spec (handles both OpenAPI 3.x and Swagger 2.0).""" + if is_swagger: + schemes = spec.get("schemes", ["https"]) + host = spec.get("host", "") + base_path = spec.get("basePath", "") + if host: + scheme = schemes[0] if schemes else "https" + return f"{scheme}://{host}{base_path}".rstrip("/") + return "" + servers = spec.get("servers", []) + if servers and isinstance(servers, list) and servers[0].get("url"): + return servers[0]["url"].rstrip("/") + return "" + + +def _extract_actions(spec: Dict[str, Any], is_swagger: bool) -> List[Dict[str, Any]]: + """Extract all API operations as action definitions.""" + actions = [] + paths = spec.get("paths", {}) + base_url = _get_base_url(spec, is_swagger) + + components = spec.get("components", {}) + definitions = spec.get("definitions", {}) + + for path, path_item in paths.items(): + if not isinstance(path_item, dict): + continue + path_params = path_item.get("parameters", []) + + for method in SUPPORTED_METHODS: + operation = path_item.get(method) + if not isinstance(operation, dict): + continue + try: + action = _build_action( + path=path, + method=method, + operation=operation, + path_params=path_params, + base_url=base_url, + components=components, + definitions=definitions, + is_swagger=is_swagger, + ) + actions.append(action) + except Exception as e: + logger.warning( + f"Failed to parse operation {method.upper()} {path}: {e}" + ) + continue + return actions + + +def _build_action( + path: str, + method: str, + operation: Dict[str, Any], + path_params: List[Dict], + base_url: str, + components: Dict[str, Any], + definitions: Dict[str, Any], + is_swagger: bool, +) -> Dict[str, Any]: + """Build a single action from an API operation.""" + action_name = _generate_action_name(operation, method, path) + full_url = f"{base_url}{path}" if base_url else path + + all_params = path_params + operation.get("parameters", []) + query_params, headers = _categorize_parameters(all_params, components, definitions) + + body, body_content_type = _extract_request_body( + operation, components, definitions, is_swagger + ) + + description = operation.get("summary", "") or operation.get("description", "") + + return { + "name": action_name, + "url": full_url, + "method": method.upper(), + "description": (description or "")[:500], + "query_params": {"type": "object", "properties": query_params}, + "headers": {"type": "object", "properties": headers}, + "body": {"type": "object", "properties": body}, + "body_content_type": body_content_type, + "active": True, + } + + +def _generate_action_name(operation: Dict[str, Any], method: str, path: str) -> str: + """Generate a valid action name from operationId or method+path.""" + if operation.get("operationId"): + name = operation["operationId"] + else: + path_slug = re.sub(r"[{}]", "", path) + path_slug = re.sub(r"[^a-zA-Z0-9]", "_", path_slug) + path_slug = re.sub(r"_+", "_", path_slug).strip("_") + name = f"{method}_{path_slug}" + name = re.sub(r"[^a-zA-Z0-9_-]", "_", name) + return name[:64] + + +def _categorize_parameters( + parameters: List[Dict], + components: Dict[str, Any], + definitions: Dict[str, Any], +) -> Tuple[Dict, Dict]: + """Categorize parameters into query params and headers.""" + query_params = {} + headers = {} + + for param in parameters: + resolved = _resolve_ref(param, components, definitions) + if not resolved or "name" not in resolved: + continue + location = resolved.get("in", "query") + prop = _param_to_property(resolved) + + if location in ("query", "path"): + query_params[resolved["name"]] = prop + elif location == "header": + headers[resolved["name"]] = prop + return query_params, headers + + +def _param_to_property(param: Dict) -> Dict[str, Any]: + """Convert an API parameter to an action property definition.""" + schema = param.get("schema", {}) + param_type = schema.get("type", param.get("type", "string")) + + mapped_type = "integer" if param_type in ("integer", "number") else "string" + + return { + "type": mapped_type, + "description": (param.get("description", "") or "")[:200], + "value": "", + "filled_by_llm": param.get("required", False), + "required": param.get("required", False), + } + + +def _extract_request_body( + operation: Dict[str, Any], + components: Dict[str, Any], + definitions: Dict[str, Any], + is_swagger: bool, +) -> Tuple[Dict, str]: + """Extract request body schema and content type.""" + content_types = [ + "application/json", + "application/x-www-form-urlencoded", + "multipart/form-data", + "text/plain", + "application/xml", + ] + + if is_swagger: + consumes = operation.get("consumes", []) + body_param = next( + (p for p in operation.get("parameters", []) if p.get("in") == "body"), None + ) + if not body_param: + return {}, "application/json" + selected_type = consumes[0] if consumes else "application/json" + schema = body_param.get("schema", {}) + else: + request_body = operation.get("requestBody", {}) + if not request_body: + return {}, "application/json" + request_body = _resolve_ref(request_body, components, definitions) + content = request_body.get("content", {}) + + selected_type = "application/json" + schema = {} + + for ct in content_types: + if ct in content: + selected_type = ct + schema = content[ct].get("schema", {}) + break + if not schema and content: + first_type = next(iter(content)) + selected_type = first_type + schema = content[first_type].get("schema", {}) + properties = _schema_to_properties(schema, components, definitions) + return properties, selected_type + + +def _schema_to_properties( + schema: Dict, + components: Dict[str, Any], + definitions: Dict[str, Any], + depth: int = 0, +) -> Dict[str, Any]: + """Convert schema to action body properties (limited depth to prevent recursion).""" + if depth > 3: + return {} + schema = _resolve_ref(schema, components, definitions) + if not schema or not isinstance(schema, dict): + return {} + properties = {} + schema_type = schema.get("type", "object") + + if schema_type == "object": + required_fields = set(schema.get("required", [])) + for prop_name, prop_schema in schema.get("properties", {}).items(): + resolved = _resolve_ref(prop_schema, components, definitions) + if not isinstance(resolved, dict): + continue + prop_type = resolved.get("type", "string") + mapped_type = "integer" if prop_type in ("integer", "number") else "string" + + properties[prop_name] = { + "type": mapped_type, + "description": (resolved.get("description", "") or "")[:200], + "value": "", + "filled_by_llm": prop_name in required_fields, + "required": prop_name in required_fields, + } + return properties + + +def _resolve_ref( + obj: Any, + components: Dict[str, Any], + definitions: Dict[str, Any], +) -> Optional[Dict]: + """Resolve $ref references in the specification.""" + if not isinstance(obj, dict): + return obj if isinstance(obj, dict) else None + if "$ref" not in obj: + return obj + ref_path = obj["$ref"] + + if ref_path.startswith("#/components/"): + parts = ref_path.replace("#/components/", "").split("/") + return _traverse_path(components, parts) + elif ref_path.startswith("#/definitions/"): + parts = ref_path.replace("#/definitions/", "").split("/") + return _traverse_path(definitions, parts) + logger.debug(f"Unsupported ref path: {ref_path}") + return None + + +def _traverse_path(obj: Dict, parts: List[str]) -> Optional[Dict]: + """Traverse a nested dictionary using path parts.""" + try: + for part in parts: + obj = obj[part] + return obj if isinstance(obj, dict) else None + except (KeyError, TypeError): + return None diff --git a/application/api/user/tools/routes.py b/application/api/user/tools/routes.py index 0d4bc6f8..1503ef7e 100644 --- a/application/api/user/tools/routes.py +++ b/application/api/user/tools/routes.py @@ -4,6 +4,7 @@ from bson.objectid import ObjectId from flask import current_app, jsonify, make_response, request from flask_restx import fields, Namespace, Resource +from application.agents.tools.spec_parser import parse_spec from application.agents.tools.tool_manager import ToolManager from application.api import api from application.api.user.base import user_tools_collection @@ -414,3 +415,57 @@ class DeleteTool(Resource): current_app.logger.error(f"Error deleting tool: {err}", exc_info=True) return {"success": False}, 400 return {"success": True}, 200 + + +@tools_ns.route("/parse_spec") +class ParseSpec(Resource): + @api.doc( + description="Parse an API specification (OpenAPI 3.x or Swagger 2.0) and return actions" + ) + def post(self): + decoded_token = request.decoded_token + if not decoded_token: + return make_response(jsonify({"success": False}), 401) + if "file" in request.files: + file = request.files["file"] + if not file.filename: + return make_response( + jsonify({"success": False, "message": "No file selected"}), 400 + ) + try: + spec_content = file.read().decode("utf-8") + except UnicodeDecodeError: + return make_response( + jsonify({"success": False, "message": "Invalid file encoding"}), 400 + ) + elif request.is_json: + data = request.get_json() + spec_content = data.get("spec_content", "") + else: + return make_response( + jsonify({"success": False, "message": "No spec provided"}), 400 + ) + if not spec_content or not spec_content.strip(): + return make_response( + jsonify({"success": False, "message": "Empty spec content"}), 400 + ) + try: + metadata, actions = parse_spec(spec_content) + return make_response( + jsonify( + { + "success": True, + "metadata": metadata, + "actions": actions, + } + ), + 200, + ) + except ValueError as e: + error_msg = str(e) + current_app.logger.error(f"Spec validation error: {error_msg}") + return make_response(jsonify({"success": False, "error": error_msg}), 400) + except Exception as err: + error_msg = str(err) + current_app.logger.error(f"Error parsing spec: {error_msg}", exc_info=True) + return make_response(jsonify({"success": False, "error": error_msg}), 500) diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index d3636dd2..6bd8a834 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -40,6 +40,7 @@ const endpoints = { UPDATE_TOOL_STATUS: '/api/update_tool_status', UPDATE_TOOL: '/api/update_tool', DELETE_TOOL: '/api/delete_tool', + PARSE_SPEC: '/api/parse_spec', SYNC_CONNECTOR: '/api/connectors/sync', GET_CHUNKS: ( docId: string, diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index ecd3df3d..1dcf9f4c 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -84,6 +84,11 @@ const userService = { apiClient.post(endpoints.USER.UPDATE_TOOL, data, token), deleteTool: (data: any, token: string | null): Promise => apiClient.post(endpoints.USER.DELETE_TOOL, data, token), + parseSpec: (file: File, token: string | null): Promise => { + const formData = new FormData(); + formData.append('file', file); + return apiClient.postFormData(endpoints.USER.PARSE_SPEC, formData, token); + }, getDocumentChunks: ( docId: string, page: number, diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json index c4b58a3e..55e10877 100644 --- a/frontend/src/locale/de.json +++ b/frontend/src/locale/de.json @@ -162,12 +162,17 @@ "authentication": "Authentifizierung", "actions": "Aktionen", "addAction": "Aktion hinzufügen", + "importSpec": "Spezifikation importieren", + "searchActions": "Aktionen suchen...", + "noActionsMatch": "Keine Aktionen passen zu deiner Suche", + "actionAlreadyExists": "Eine Aktion mit diesem Namen existiert bereits", "noActionsFound": "Keine Aktionen gefunden", "url": "URL", "urlPlaceholder": "URL eingeben", "method": "Methode", "description": "Beschreibung", "descriptionPlaceholder": "Beschreibung eingeben", + "bodyContentType": "Body-Inhaltstyp", "headers": "Header", "queryParameters": "Abfrageparameter", "body": "Body", @@ -441,6 +446,22 @@ "generate": "Generieren", "test": "Testen", "learnMore": "Mehr erfahren" + }, + "importSpec": { + "title": "API-Spezifikation importieren", + "description": "Lade eine OpenAPI 3.x- oder Swagger 2.0-Spezifikationsdatei hoch, um automatisch Aktionen zu generieren.", + "dropzoneText": "Zum Hochladen klicken oder per Drag & Drop", + "supportedFormats": "JSON- oder YAML-Format", + "invalidFileType": "Ungültiger Dateityp. Bitte eine JSON- oder YAML-Datei hochladen.", + "parseError": "Spezifikation konnte nicht geparst werden. Bitte Dateiformat prüfen.", + "version": "Version", + "baseUrl": "Basis-URL", + "actionsFound": "{{count}} Aktionen gefunden", + "selectAll": "Alle auswählen", + "deselectAll": "Alle abwählen", + "cancel": "Abbrechen", + "parse": "Parsen", + "import": "Importieren ({{count}})" } }, "sharedConv": { diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index dbb8bdbe..94680a96 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -162,12 +162,17 @@ "authentication": "Authentication", "actions": "Actions", "addAction": "Add action", + "importSpec": "Import Spec", + "searchActions": "Search actions...", + "noActionsMatch": "No actions match your search", + "actionAlreadyExists": "An action with this name already exists", "noActionsFound": "No actions found", "url": "URL", "urlPlaceholder": "Enter URL", "method": "Method", "description": "Description", "descriptionPlaceholder": "Enter description", + "bodyContentType": "Body Content Type", "headers": "Headers", "queryParameters": "Query Parameters", "body": "Body", @@ -441,6 +446,22 @@ "generate": "Generate", "test": "Test", "learnMore": "Learn more" + }, + "importSpec": { + "title": "Import API Specification", + "description": "Upload an OpenAPI 3.x or Swagger 2.0 specification file to automatically generate actions.", + "dropzoneText": "Click to upload or drag and drop", + "supportedFormats": "JSON or YAML format", + "invalidFileType": "Invalid file type. Please upload a JSON or YAML file.", + "parseError": "Failed to parse the specification. Please check the file format.", + "version": "Version", + "baseUrl": "Base URL", + "actionsFound": "{{count}} actions found", + "selectAll": "Select all", + "deselectAll": "Deselect all", + "cancel": "Cancel", + "parse": "Parse", + "import": "Import ({{count}})" } }, "sharedConv": { diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index 49f09344..f388796e 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -162,12 +162,17 @@ "authentication": "Autenticación", "actions": "Acciones", "addAction": "Agregar acción", + "importSpec": "Importar especificación", + "searchActions": "Buscar acciones...", + "noActionsMatch": "No hay acciones que coincidan con tu búsqueda", + "actionAlreadyExists": "Ya existe una acción con este nombre", "noActionsFound": "No se encontraron acciones", "url": "URL", "urlPlaceholder": "Ingresa url", "method": "Método", "description": "Descripción", "descriptionPlaceholder": "Ingresa descripción", + "bodyContentType": "Tipo de contenido del cuerpo", "headers": "Encabezados", "queryParameters": "Parámetros de Consulta", "body": "Cuerpo", @@ -441,6 +446,22 @@ "generate": "Generate", "test": "Test", "learnMore": "Learn more" + }, + "importSpec": { + "title": "Importar especificación de API", + "description": "Sube un archivo de especificación OpenAPI 3.x o Swagger 2.0 para generar acciones automáticamente.", + "dropzoneText": "Haz clic para subir o arrastra y suelta", + "supportedFormats": "Formato JSON o YAML", + "invalidFileType": "Tipo de archivo no válido. Sube un archivo JSON o YAML.", + "parseError": "No se pudo analizar la especificación. Verifica el formato del archivo.", + "version": "Versión", + "baseUrl": "URL base", + "actionsFound": "{{count}} acciones encontradas", + "selectAll": "Seleccionar todo", + "deselectAll": "Deseleccionar todo", + "cancel": "Cancelar", + "parse": "Analizar", + "import": "Importar ({{count}})" } }, "sharedConv": { diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index b8458d2c..cc5a9de3 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -162,12 +162,17 @@ "authentication": "認証", "actions": "アクション", "addAction": "アクションを追加", + "importSpec": "仕様をインポート", + "searchActions": "アクションを検索...", + "noActionsMatch": "検索に一致するアクションがありません", + "actionAlreadyExists": "この名前のアクションは既に存在します", "noActionsFound": "アクションが見つかりません", "url": "URL", "urlPlaceholder": "URLを入力", "method": "メソッド", "description": "説明", "descriptionPlaceholder": "説明を入力", + "bodyContentType": "ボディのコンテンツタイプ", "headers": "ヘッダー", "queryParameters": "クエリパラメータ", "body": "ボディ", @@ -441,6 +446,22 @@ "generate": "Generate", "test": "Test", "learnMore": "Learn more" + }, + "importSpec": { + "title": "API仕様のインポート", + "description": "OpenAPI 3.x または Swagger 2.0 の仕様ファイルをアップロードして、アクションを自動生成します。", + "dropzoneText": "クリックしてアップロード、またはドラッグ＆ドロップ", + "supportedFormats": "JSON または YAML 形式", + "invalidFileType": "無効なファイル形式です。JSON または YAML ファイルをアップロードしてください。", + "parseError": "仕様の解析に失敗しました。ファイル形式を確認してください。", + "version": "バージョン", + "baseUrl": "ベースURL", + "actionsFound": "{{count}} 件のアクションが見つかりました", + "selectAll": "すべて選択", + "deselectAll": "すべて解除", + "cancel": "キャンセル", + "parse": "解析", + "import": "インポート ({{count}})" } }, "sharedConv": { diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index 103f3627..d9146781 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -162,12 +162,17 @@ "authentication": "Аутентификация", "actions": "Действия", "addAction": "Добавить действие", + "importSpec": "Импорт спецификации", + "searchActions": "Поиск действий...", + "noActionsMatch": "Нет действий, соответствующих вашему поиску", + "actionAlreadyExists": "Действие с таким именем уже существует", "noActionsFound": "Действия не найдены", "url": "URL", "urlPlaceholder": "Введите URL", "method": "Метод", "description": "Описание", "descriptionPlaceholder": "Введите описание", + "bodyContentType": "Тип содержимого тела", "headers": "Заголовки", "queryParameters": "Параметры запроса", "body": "Тело запроса", @@ -441,6 +446,22 @@ "generate": "Generate", "test": "Test", "learnMore": "Learn more" + }, + "importSpec": { + "title": "Импорт спецификации API", + "description": "Загрузите файл спецификации OpenAPI 3.x или Swagger 2.0 для автоматического создания действий.", + "dropzoneText": "Нажмите для загрузки или перетащите файл", + "supportedFormats": "Формат JSON или YAML", + "invalidFileType": "Неверный тип файла. Пожалуйста, загрузите файл JSON или YAML.", + "parseError": "Не удалось разобрать спецификацию. Проверьте формат файла.", + "version": "Версия", + "baseUrl": "Базовый URL", + "actionsFound": "{{count}} действий найдено", + "selectAll": "Выбрать все", + "deselectAll": "Снять выделение со всех", + "cancel": "Отмена", + "parse": "Разобрать", + "import": "Импорт ({{count}})" } }, "sharedConv": { diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index 47c0b236..8e8d9714 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -162,12 +162,17 @@ "authentication": "認證", "actions": "操作", "addAction": "新增操作", + "importSpec": "匯入規格", + "searchActions": "搜尋操作...", + "noActionsMatch": "沒有符合搜尋的操作", + "actionAlreadyExists": "已存在同名操作", "noActionsFound": "找不到操作", "url": "URL", "urlPlaceholder": "輸入url", "method": "方法", "description": "描述", "descriptionPlaceholder": "輸入描述", + "bodyContentType": "主體內容類型", "headers": "標頭", "queryParameters": "查詢參數", "body": "主體", @@ -441,6 +446,22 @@ "generate": "Generate", "test": "Test", "learnMore": "Learn more" + }, + "importSpec": { + "title": "匯入 API 規格", + "description": "上傳 OpenAPI 3.x 或 Swagger 2.0 規格檔以自動產生操作。", + "dropzoneText": "點擊上傳或拖放", + "supportedFormats": "JSON 或 YAML 格式", + "invalidFileType": "無效的檔案類型。請上傳 JSON 或 YAML 檔案。", + "parseError": "解析規格失敗。請檢查檔案格式。", + "version": "版本", + "baseUrl": "基礎 URL", + "actionsFound": "找到 {{count}} 個操作", + "selectAll": "全選", + "deselectAll": "取消全選", + "cancel": "取消", + "parse": "解析", + "import": "匯入 ({{count}})" } }, "sharedConv": { diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 12b867fa..f85cef5b 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -162,12 +162,17 @@ "authentication": "认证", "actions": "操作", "addAction": "添加操作", + "importSpec": "导入规范", + "searchActions": "搜索操作...", + "noActionsMatch": "没有与搜索匹配的操作", + "actionAlreadyExists": "已存在同名操作", "noActionsFound": "未找到操作", "url": "URL", "urlPlaceholder": "输入url", "method": "方法", "description": "描述", "descriptionPlaceholder": "输入描述", + "bodyContentType": "请求体内容类型", "headers": "请求头", "queryParameters": "查询参数", "body": "请求体", @@ -441,6 +446,22 @@ "generate": "Generate", "test": "Test", "learnMore": "Learn more" + }, + "importSpec": { + "title": "导入 API 规范", + "description": "上传 OpenAPI 3.x 或 Swagger 2.0 规范文件以自动生成操作。", + "dropzoneText": "点击上传或拖拽到此处", + "supportedFormats": "JSON 或 YAML 格式", + "invalidFileType": "文件类型无效。请上传 JSON 或 YAML 文件。", + "parseError": "解析规范失败。请检查文件格式。", + "version": "版本", + "baseUrl": "基础 URL", + "actionsFound": "找到 {{count}} 个操作", + "selectAll": "全选", + "deselectAll": "取消全选", + "cancel": "取消", + "parse": "解析", + "import": "导入 ({{count}})" } }, "sharedConv": { diff --git a/frontend/src/modals/ImportSpecModal.tsx b/frontend/src/modals/ImportSpecModal.tsx new file mode 100644 index 00000000..df68aea6 --- /dev/null +++ b/frontend/src/modals/ImportSpecModal.tsx @@ -0,0 +1,321 @@ +import { useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useSelector } from 'react-redux'; + +import userService from '../api/services/userService'; +import Upload from '../assets/upload.svg'; +import Spinner from '../components/Spinner'; +import { ActiveState } from '../models/misc'; +import { selectToken } from '../preferences/preferenceSlice'; +import { APIActionType } from '../settings/types'; +import WrapperModal from './WrapperModal'; + +interface ImportSpecModalProps { + modalState: ActiveState; + setModalState: (state: ActiveState) => void; + onImport: (actions: APIActionType[]) => void; +} + +interface ParsedResult { + metadata: { + title: string; + description: string; + version: string; + base_url: string; + }; + actions: APIActionType[]; +} + +const METHOD_COLORS: Record = { + GET: 'bg-[#D1FAE5] text-[#065F46] dark:bg-[#064E3B]/60 dark:text-[#6EE7B7]', + POST: 'bg-[#DBEAFE] text-[#1E40AF] dark:bg-[#1E3A8A]/60 dark:text-[#93C5FD]', + PUT: 'bg-[#FEF3C7] text-[#92400E] dark:bg-[#78350F]/60 dark:text-[#FCD34D]', + DELETE: + 'bg-[#FEE2E2] text-[#991B1B] dark:bg-[#7F1D1D]/60 dark:text-[#FCA5A5]', + PATCH: 'bg-[#EDE9FE] text-[#5B21B6] dark:bg-[#4C1D95]/60 dark:text-[#C4B5FD]', + HEAD: 'bg-[#F3F4F6] text-[#374151] dark:bg-[#374151]/60 dark:text-[#D1D5DB]', + OPTIONS: + 'bg-[#F3F4F6] text-[#374151] dark:bg-[#374151]/60 dark:text-[#D1D5DB]', +}; + +export default function ImportSpecModal({ + modalState, + setModalState, + onImport, +}: ImportSpecModalProps) { + const { t } = useTranslation(); + const token = useSelector(selectToken); + const fileInputRef = useRef(null); + + const [file, setFile] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [parsedResult, setParsedResult] = useState(null); + const [selectedActions, setSelectedActions] = useState>( + new Set(), + ); + const [baseUrl, setBaseUrl] = useState(''); + + const handleClose = () => { + setModalState('INACTIVE'); + setFile(null); + setLoading(false); + setError(null); + setParsedResult(null); + setSelectedActions(new Set()); + setBaseUrl(''); + }; + + const handleFileChange = (e: React.ChangeEvent) => { + const selectedFile = e.target.files?.[0]; + if (!selectedFile) return; + + const validExtensions = ['.json', '.yaml', '.yml']; + const hasValidExtension = validExtensions.some((ext) => + selectedFile.name.toLowerCase().endsWith(ext), + ); + + if (!hasValidExtension) { + setError(t('modals.importSpec.invalidFileType')); + return; + } + + setFile(selectedFile); + setError(null); + setParsedResult(null); + }; + + const handleParse = async () => { + if (!file) return; + + setLoading(true); + setError(null); + + try { + const response = await userService.parseSpec(file, token); + if (!response.ok) { + const errorData = await response.json(); + setError( + errorData.error || + errorData.message || + t('modals.importSpec.parseError'), + ); + return; + } + + const result = await response.json(); + if (result.success) { + setParsedResult(result); + setBaseUrl(result.metadata.base_url || ''); + setSelectedActions( + new Set( + result.actions.map((_: APIActionType, i: number) => i), + ), + ); + } else { + setError( + result.error || result.message || t('modals.importSpec.parseError'), + ); + } + } catch { + setError(t('modals.importSpec.parseError')); + } finally { + setLoading(false); + } + }; + + const toggleAction = (index: number) => { + setSelectedActions((prev) => { + const next = new Set(prev); + if (next.has(index)) { + next.delete(index); + } else { + next.add(index); + } + return next; + }); + }; + + const toggleAll = () => { + if (!parsedResult) return; + if (selectedActions.size === parsedResult.actions.length) { + setSelectedActions(new Set()); + } else { + setSelectedActions(new Set(parsedResult.actions.map((_, i) => i))); + } + }; + + const handleImport = () => { + if (!parsedResult) return; + const actionsToImport = parsedResult.actions + .filter((_, i) => selectedActions.has(i)) + .map((action) => ({ + ...action, + url: action.url.replace(parsedResult.metadata.base_url, baseUrl.trim()), + })); + onImport(actionsToImport); + handleClose(); + }; + + if (modalState !== 'ACTIVE') return null; + + return ( + +
+

+ {t('modals.importSpec.title')} +

+ + {!parsedResult ? ( +
+

+ {t('modals.importSpec.description')} +

+ +
fileInputRef.current?.click()} + className="border-silver dark:border-silver/40 hover:border-purple-30 dark:hover:border-purple-30 flex cursor-pointer flex-col items-center justify-center rounded-xl border-2 border-dashed p-8 transition-colors" + > + Upload +

+ {file ? file.name : t('modals.importSpec.dropzoneText')} +

+

+ {t('modals.importSpec.supportedFormats')} +

+ +
+ + {error && ( +

{error}

+ )} +
+ ) : ( +
+
+

+ {parsedResult.metadata.title} +

+ {parsedResult.metadata.description && ( +

+ {parsedResult.metadata.description} +

+ )} +

+ {t('modals.importSpec.version')}:{' '} + {parsedResult.metadata.version} +

+
+ + setBaseUrl(e.target.value)} + className="border-silver dark:border-silver/40 text-jet dark:text-bright-gray w-full rounded-lg border bg-white px-3 py-2 text-sm outline-hidden dark:bg-[#2C2C2C]" + placeholder={ + parsedResult.metadata.base_url || 'https://api.example.com' + } + /> +
+
+ +
+

+ {t('modals.importSpec.actionsFound', { + count: parsedResult.actions.length, + })} +

+ +
+ +
+ {parsedResult.actions.map((action, index) => ( + + ))} +
+
+ )} + +
+ {!parsedResult ? ( + + ) : ( + + )} + +
+
+ + ); +} diff --git a/frontend/src/settings/ToolConfig.tsx b/frontend/src/settings/ToolConfig.tsx index bca5c6ce..b6177483 100644 --- a/frontend/src/settings/ToolConfig.tsx +++ b/frontend/src/settings/ToolConfig.tsx @@ -1,24 +1,38 @@ import React from 'react'; +import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; import userService from '../api/services/userService'; import ArrowLeft from '../assets/arrow-left.svg'; +import ChevronRight from '../assets/chevron-right.svg'; import CircleCheck from '../assets/circle-check.svg'; import CircleX from '../assets/circle-x.svg'; +import NoFilesDarkIcon from '../assets/no-files-dark.svg'; +import NoFilesIcon from '../assets/no-files.svg'; import Trash from '../assets/trash.svg'; import Dropdown from '../components/Dropdown'; import Input from '../components/Input'; import ToggleSwitch from '../components/ToggleSwitch'; +import { useDarkTheme } from '../hooks'; import AddActionModal from '../modals/AddActionModal'; import ConfirmationModal from '../modals/ConfirmationModal'; +import ImportSpecModal from '../modals/ImportSpecModal'; import { ActiveState } from '../models/misc'; import { selectToken } from '../preferences/preferenceSlice'; -import { APIActionType, APIToolType, UserToolType } from './types'; -import { useTranslation } from 'react-i18next'; import { areObjectsEqual } from '../utils/objectUtils'; -import { useDarkTheme } from '../hooks'; -import NoFilesIcon from '../assets/no-files.svg'; -import NoFilesDarkIcon from '../assets/no-files-dark.svg'; +import { APIActionType, APIToolType, UserToolType } from './types'; + +const METHOD_COLORS: Record = { + GET: 'bg-[#D1FAE5] text-[#065F46] dark:bg-[#064E3B]/60 dark:text-[#6EE7B7]', + POST: 'bg-[#DBEAFE] text-[#1E40AF] dark:bg-[#1E3A8A]/60 dark:text-[#93C5FD]', + PUT: 'bg-[#FEF3C7] text-[#92400E] dark:bg-[#78350F]/60 dark:text-[#FCD34D]', + DELETE: + 'bg-[#FEE2E2] text-[#991B1B] dark:bg-[#7F1D1D]/60 dark:text-[#FCA5A5]', + PATCH: 'bg-[#EDE9FE] text-[#5B21B6] dark:bg-[#4C1D95]/60 dark:text-[#C4B5FD]', + HEAD: 'bg-[#F3F4F6] text-[#374151] dark:bg-[#374151]/60 dark:text-[#D1D5DB]', + OPTIONS: + 'bg-[#F3F4F6] text-[#374151] dark:bg-[#374151]/60 dark:text-[#D1D5DB]', +}; export default function ToolConfig({ tool, @@ -51,6 +65,8 @@ export default function ToolConfig({ ); const [actionModalState, setActionModalState] = React.useState('INACTIVE'); + const [importModalState, setImportModalState] = + React.useState('INACTIVE'); const [initialState, setInitialState] = React.useState({ customName: tool.customName || '', authKey: 'token' in tool.config ? tool.config.token : '', @@ -59,9 +75,38 @@ export default function ToolConfig({ }); const [hasUnsavedChanges, setHasUnsavedChanges] = React.useState(false); const [showUnsavedModal, setShowUnsavedModal] = React.useState(false); + const [userActionsSearch, setUserActionsSearch] = React.useState(''); + const [expandedUserActions, setExpandedUserActions] = React.useState< + Set + >(new Set()); const { t } = useTranslation(); const [isDarkTheme] = useDarkTheme(); + const toggleUserActionExpand = (index: number) => { + setExpandedUserActions((prev) => { + const newSet = new Set(prev); + if (newSet.has(index)) { + newSet.delete(index); + } else { + newSet.add(index); + } + return newSet; + }); + }; + + const filteredUserActions = React.useMemo(() => { + if (!('actions' in tool) || !tool.actions) return []; + const query = userActionsSearch.toLowerCase(); + return tool.actions + .map((action, index) => ({ action, originalIndex: index })) + .filter( + ({ action }) => + action.name.toLowerCase().includes(query) || + action.description?.toLowerCase().includes(query), + ) + .sort((a, b) => a.action.name.localeCompare(b.action.name)); + }, [tool, userActionsSearch]); + const handleBackClick = () => { if (hasUnsavedChanges) { setShowUnsavedModal(true); @@ -88,6 +133,8 @@ export default function ToolConfig({ 'actions' in tool ? tool.actions.map((action, index) => { if (index === actionIndex) { + const newFilledByLlm = + !action.parameters.properties[property].filled_by_llm; return { ...action, parameters: { @@ -96,8 +143,8 @@ export default function ToolConfig({ ...action.parameters.properties, [property]: { ...action.parameters.properties[property], - filled_by_llm: - !action.parameters.properties[property].filled_by_llm, + filled_by_llm: newFilledByLlm, + required: newFilledByLlm, }, }, }, @@ -164,6 +211,13 @@ export default function ToolConfig({ }; const handleAddNewAction = (actionName: string) => { + const toolCopy = tool as APIToolType; + + if (toolCopy.config.actions && toolCopy.config.actions[actionName]) { + alert(t('settings.tools.actionAlreadyExists')); + return; + } + const newAction: APIActionType = { name: actionName, method: 'GET', @@ -182,8 +236,10 @@ export default function ToolConfig({ type: 'object', }, active: true, + body_content_type: 'application/json', + body_encoding_rules: {}, }; - const toolCopy = tool as APIToolType; + setTool({ ...toolCopy, config: { @@ -192,6 +248,30 @@ export default function ToolConfig({ }, }); }; + + const handleImportActions = (actions: APIActionType[]) => { + const toolCopy = tool as APIToolType; + const existingActions = toolCopy.config.actions || {}; + const newActions: { [key: string]: APIActionType } = {}; + + actions.forEach((action) => { + let actionName = action.name; + let counter = 1; + while (existingActions[actionName] || newActions[actionName]) { + actionName = `${action.name}_${counter}`; + counter++; + } + newActions[actionName] = { ...action, name: actionName }; + }); + + setTool({ + ...toolCopy, + config: { + ...toolCopy.config, + actions: { ...existingActions, ...newActions }, + }, + }); + }; return (
@@ -271,16 +351,22 @@ export default function ToolConfig({

{t('settings.tools.actions')}

- {tool.name === 'api_tool' && - (!tool.config.actions || - Object.keys(tool.config.actions).length === 0) && ( + {tool.name === 'api_tool' && ( +
+ - )} +
+ )}
{tool.name === 'api_tool' ? ( <> @@ -301,180 +387,247 @@ export default function ToolConfig({ )} ) : ( -
+
{'actions' in tool && tool.actions && tool.actions.length > 0 ? ( - tool.actions.map((action, actionIndex) => ( -
-
-

- {action.name} -

- { - setTool({ - ...tool, - actions: tool.actions.map((act, index) => { - if (index === actionIndex) { - return { ...act, active: checked }; - } - return act; - }), - }); - }} - size="small" - id={`actionToggle-${actionIndex}`} + <> +
+ setUserActionsSearch(e.target.value)} + placeholder={t('settings.tools.searchActions')} + className="border-silver dark:border-silver/40 dark:bg-raisin-black w-full rounded-full border px-4 py-2 pl-10 text-sm outline-none focus:border-purple-500 dark:text-white dark:placeholder-gray-500" + /> + + -
-
- { - setTool({ - ...tool, - actions: tool.actions.map((act, index) => { - if (index === actionIndex) { - return { - ...act, - description: e.target.value, - }; - } - return act; - }), - }); - }} - borderVariant="thin" - /> -
-
- - - - - - - - - - - - {Object.entries(action.parameters?.properties).map( - (param, index) => { - const uniqueKey = `${actionIndex}-${param[0]}`; - return ( - - - - - - - - ); - }, - )} - -
{t('settings.tools.fieldName')}{t('settings.tools.fieldType')}{t('settings.tools.filledByLLM')}{t('settings.tools.fieldDescription')}{t('settings.tools.value')}
{param[0]}{param[1].type} - - - { - setTool({ - ...tool, - actions: tool.actions.map( - (act, index) => { - if (index === actionIndex) { - return { - ...act, - parameters: { - ...act.parameters, - properties: { - ...act.parameters - .properties, - [param[0]]: { - ...act.parameters - .properties[param[0]], - description: - e.target.value, - }, - }, - }, - }; - } - return act; - }, - ), - }); - }} - > - - { - setTool({ - ...tool, - actions: tool.actions.map( - (act, index) => { - if (index === actionIndex) { - return { - ...act, - parameters: { - ...act.parameters, - properties: { - ...act.parameters - .properties, - [param[0]]: { - ...act.parameters - .properties[param[0]], - value: e.target.value, - }, - }, - }, - }; - } - return act; - }, - ), - }); - }} - > -
-
+
- )) + + {filteredUserActions.length === 0 && userActionsSearch && ( +

+ {t('settings.tools.noActionsMatch')} +

+ )} + + {filteredUserActions.map(({ action, originalIndex }) => { + const isExpanded = expandedUserActions.has(originalIndex); + return ( +
+
toggleUserActionExpand(originalIndex)} + > +
+ expand +

+ {action.name} +

+ {action.description && ( +

+ {action.description} +

+ )} +
+
e.stopPropagation()} + > + { + setTool({ + ...tool, + actions: tool.actions.map((act, index) => { + if (index === originalIndex) { + return { ...act, active: checked }; + } + return act; + }), + }); + }} + size="small" + id={`actionToggle-${originalIndex}`} + /> +
+
+ {isExpanded && ( + <> +
+ { + setTool({ + ...tool, + actions: tool.actions.map((act, index) => { + if (index === originalIndex) { + return { + ...act, + description: e.target.value, + }; + } + return act; + }), + }); + }} + borderVariant="thin" + /> +
+
+ + + + + + + + + + + + {Object.entries( + action.parameters?.properties, + ).map((param, paramIndex) => { + const uniqueKey = `${originalIndex}-${param[0]}`; + return ( + + + + + + + + ); + })} + +
{t('settings.tools.fieldName')}{t('settings.tools.fieldType')}{t('settings.tools.filledByLLM')} + {t('settings.tools.fieldDescription')} + {t('settings.tools.value')}
{param[0]}{param[1].type} + + + { + setTool({ + ...tool, + actions: tool.actions.map( + (act, index) => { + if (index === originalIndex) { + return { + ...act, + parameters: { + ...act.parameters, + properties: { + ...act.parameters + .properties, + [param[0]]: { + ...act.parameters + .properties[ + param[0] + ], + description: + e.target.value, + }, + }, + }, + }; + } + return act; + }, + ), + }); + }} + > + + { + setTool({ + ...tool, + actions: tool.actions.map( + (act, index) => { + if (index === originalIndex) { + return { + ...act, + parameters: { + ...act.parameters, + properties: { + ...act.parameters + .properties, + [param[0]]: { + ...act.parameters + .properties[ + param[0] + ], + value: + e.target.value, + }, + }, + }, + }; + } + return act; + }, + ), + }); + }} + > +
+
+ + )} +
+ ); + })} + ) : (
+ {showUnsavedModal && ( ('INACTIVE'); + const [searchQuery, setSearchQuery] = React.useState(''); + const [expandedActions, setExpandedActions] = React.useState>( + new Set(), + ); + + const toggleActionExpand = (actionName: string) => { + setExpandedActions((prev) => { + const newSet = new Set(prev); + if (newSet.has(actionName)) { + newSet.delete(actionName); + } else { + newSet.add(actionName); + } + return newSet; + }); + }; + + const filteredActions = React.useMemo(() => { + if (!apiTool.config.actions) return []; + const entries = Object.entries(apiTool.config.actions); + const filtered = entries.filter(([actionName, action]) => { + const query = searchQuery.toLowerCase(); + return ( + actionName.toLowerCase().includes(query) || + action.name.toLowerCase().includes(query) || + action.description?.toLowerCase().includes(query) || + action.url?.toLowerCase().includes(query) + ); + }); + return filtered.sort((a, b) => a[0].localeCompare(b[0])); + }, [apiTool.config.actions, searchQuery]); const handleDeleteActionClick = (actionName: string) => { setActionToDelete(actionName); @@ -623,21 +812,78 @@ function APIToolConfig({ React.useEffect(() => { setTool(apiTool); }, [apiTool]); + + const getMethodColor = (method: string) => { + return METHOD_COLORS[method.toUpperCase()] || METHOD_COLORS.GET; + }; + return ( -
- {/* Actions list */} - {apiTool.config.actions && - Object.entries(apiTool.config.actions).map( - ([actionName, action], actionIndex) => ( +
+
+ setSearchQuery(e.target.value)} + placeholder={t('settings.tools.searchActions')} + className="border-silver dark:border-silver/40 dark:bg-raisin-black w-full rounded-full border px-4 py-2 pl-10 text-sm outline-none focus:border-purple-500 dark:text-white dark:placeholder-gray-500" + /> + + + +
+ + {filteredActions.length === 0 && searchQuery && ( +

+ {t('settings.tools.noActionsMatch')} +

+ )} + +
+ {filteredActions.map(([actionName, action], actionIndex) => { + const isExpanded = expandedActions.has(actionName); + return (
-
-

- {action.name} -

-
+
toggleActionExpand(actionName)} + > +
+ expand + + {action.method} + +

+ {action.name} +

+ {action.description && ( +

+ {action.description} +

+ )} +
+
e.stopPropagation()} + >
-
- { - setApiTool((prevApiTool) => { - const updatedActions = { - ...prevApiTool.config.actions, - }; - const updatedAction = { - ...updatedActions[actionName], - }; - updatedAction.url = e.target.value; - updatedActions[actionName] = updatedAction; - return { - ...prevApiTool, - config: { - ...prevApiTool.config, - actions: updatedActions, - }, - }; - }); - }} - borderVariant="thin" - placeholder={t('settings.tools.urlPlaceholder')} - /> -
-
-
- - {t('settings.tools.method')} - - { - setApiTool((prevApiTool) => { - const updatedActions = { - ...prevApiTool.config.actions, - }; - const updatedAction = { - ...updatedActions[actionName], - }; - updatedAction.method = value as - | 'GET' - | 'POST' - | 'PUT' - | 'DELETE'; - updatedActions[actionName] = updatedAction; - return { - ...prevApiTool, - config: { - ...prevApiTool.config, - actions: updatedActions, - }, - }; - }); - }} - size="w-56" - rounded="3xl" - border="border" - /> -
-
-
- { - setApiTool((prevApiTool) => { - const updatedActions = { - ...prevApiTool.config.actions, - }; - const updatedAction = { - ...updatedActions[actionName], - }; - updatedAction.description = e.target.value; - updatedActions[actionName] = updatedAction; - return { - ...prevApiTool, - config: { - ...prevApiTool.config, - actions: updatedActions, - }, - }; - }); - }} - borderVariant="thin" - placeholder={t('settings.tools.descriptionPlaceholder')} - /> -
-
- -
+ {isExpanded && ( + <> +
+ { + setApiTool((prevApiTool) => { + const updatedActions = { + ...prevApiTool.config.actions, + }; + const updatedAction = { + ...updatedActions[actionName], + }; + updatedAction.url = e.target.value; + updatedActions[actionName] = updatedAction; + return { + ...prevApiTool, + config: { + ...prevApiTool.config, + actions: updatedActions, + }, + }; + }); + }} + borderVariant="thin" + placeholder={t('settings.tools.urlPlaceholder')} + /> +
+
+
+ + {t('settings.tools.method')} + + { + setApiTool((prevApiTool) => { + const updatedActions = { + ...prevApiTool.config.actions, + }; + const updatedAction = { + ...updatedActions[actionName], + }; + updatedAction.method = value as + | 'GET' + | 'POST' + | 'PUT' + | 'DELETE' + | 'PATCH' + | 'HEAD' + | 'OPTIONS'; + updatedActions[actionName] = updatedAction; + return { + ...prevApiTool, + config: { + ...prevApiTool.config, + actions: updatedActions, + }, + }; + }); + }} + size="w-56" + rounded="3xl" + border="border" + /> +
+
+
+ { + setApiTool((prevApiTool) => { + const updatedActions = { + ...prevApiTool.config.actions, + }; + const updatedAction = { + ...updatedActions[actionName], + }; + updatedAction.description = e.target.value; + updatedActions[actionName] = updatedAction; + return { + ...prevApiTool, + config: { + ...prevApiTool.config, + actions: updatedActions, + }, + }; + }); + }} + borderVariant="thin" + placeholder={t('settings.tools.descriptionPlaceholder')} + /> +
+ {(action.method === 'POST' || + action.method === 'PUT' || + action.method === 'PATCH' || + action.method === 'HEAD' || + action.method === 'OPTIONS') && ( +
+
+ + {t('settings.tools.bodyContentType')} + + { + setApiTool((prevApiTool) => { + const updatedActions = { + ...prevApiTool.config.actions, + }; + const updatedAction = { + ...updatedActions[actionName], + }; + updatedAction.body_content_type = value as + | 'application/json' + | 'application/x-www-form-urlencoded' + | 'multipart/form-data' + | 'text/plain' + | 'application/xml' + | 'application/octet-stream'; + updatedActions[actionName] = updatedAction; + return { + ...prevApiTool, + config: { + ...prevApiTool.config, + actions: updatedActions, + }, + }; + }); + }} + size="w-56" + rounded="3xl" + border="border" + /> +
+

+ {action.body_content_type === 'multipart/form-data' && + 'For APIs requiring multipart format. File uploads not supported through LLM.'} + {action.body_content_type === + 'application/octet-stream' && + 'Raw binary data, base64-encoded for transmission.'} + {action.body_content_type === + 'application/x-www-form-urlencoded' && + 'Standard form submission format. Best for legacy APIs and login forms.'} + {action.body_content_type === 'application/xml' && + 'Structured XML format. Use for SOAP and enterprise APIs.'} + {action.body_content_type === 'text/plain' && + 'Raw text data. Each field on a new line.'} + {(!action.body_content_type || + action.body_content_type === 'application/json') && + 'Most common format. Use for modern REST APIs.'} +

+
+ )} +
+ +
+ + )}
- ), - )} + ); + })} +
{/* Confirmation Modal */} {deleteModalState === 'ACTIVE' && actionToDelete && ( @@ -793,6 +1126,9 @@ function APIActionTable({ const [action, setAction] = React.useState(apiAction); const [newPropertyKey, setNewPropertyKey] = React.useState(''); + const [newPropertyType, setNewPropertyType] = React.useState< + 'string' | 'integer' + >('string'); const [addingPropertySection, setAddingPropertySection] = React.useState< 'headers' | 'query_params' | 'body' | null >(null); @@ -808,12 +1144,17 @@ function APIActionTable({ value: string | number | boolean, ) => { setAction((prevAction) => { + const currentProperty = prevAction[section].properties[key]; + const updatedProperty: typeof currentProperty = { + ...currentProperty, + [field]: value, + ...(field === 'filled_by_llm' && typeof value === 'boolean' + ? { required: value } + : {}), + }; const updatedProperties = { ...prevAction[section].properties, - [key]: { - ...prevAction[section].properties[key], - [field]: value, - }, + [key]: updatedProperty, }; return { ...prevAction, @@ -831,10 +1172,12 @@ function APIActionTable({ setEditingPropertyKey({ section: null, oldKey: null }); setAddingPropertySection(section); setNewPropertyKey(''); + setNewPropertyType('string'); }; const handleAddPropertyCancel = () => { setAddingPropertySection(null); setNewPropertyKey(''); + setNewPropertyType('string'); }; const handleAddProperty = () => { if (addingPropertySection && newPropertyKey.trim() !== '') { @@ -842,10 +1185,11 @@ function APIActionTable({ const updatedProperties = { ...prevAction[addingPropertySection].properties, [newPropertyKey.trim()]: { - type: 'string', + type: newPropertyType, description: '', value: '', filled_by_llm: false, + required: false, }, }; return { @@ -857,6 +1201,7 @@ function APIActionTable({ }; }); setNewPropertyKey(''); + setNewPropertyType('string'); setAddingPropertySection(null); } }; @@ -872,6 +1217,7 @@ function APIActionTable({ const handleRenamePropertyCancel = () => { setEditingPropertyKey({ section: null, oldKey: null }); setNewPropertyKey(''); + setNewPropertyType('string'); }; const handleRenameProperty = () => { if ( @@ -901,6 +1247,7 @@ function APIActionTable({ }); setEditingPropertyKey({ section: null, oldKey: null }); setNewPropertyKey(''); + setNewPropertyType('string'); } }; @@ -921,6 +1268,29 @@ function APIActionTable({ }); }; + const handlePropertyTypeChange = ( + section: 'headers' | 'query_params' | 'body', + key: string, + newType: 'string' | 'integer', + ) => { + setAction((prevAction) => { + const updatedProperties = { + ...prevAction[section].properties, + [key]: { + ...prevAction[section].properties[key], + type: newType, + }, + }; + return { + ...prevAction, + [section]: { + ...prevAction[section], + properties: updatedProperties, + }, + }; + }); + }; + React.useEffect(() => { setAction(apiAction); }, [apiAction]); @@ -978,7 +1348,22 @@ function APIActionTable({ /> )} - {param.type} + + +
+ ) : ( + handleRenamePropertyStart('headers', key)} + readOnly + /> + )} + + + + handlePropertyChange( + 'headers', + key, + 'value', + e.target.value, + ) + } + placeholder="e.g., application/json" + className="border-silver dark:border-silver/40 w-full rounded-lg border bg-transparent px-2 py-1 text-sm outline-hidden" + /> + + + + handlePropertyChange( + 'headers', + key, + 'description', + e.target.value, + ) + } + /> + + + + + + ), + )} + {addingPropertySection === 'headers' ? ( + + + setNewPropertyKey(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + handleAddProperty(); + } + }} + placeholder={t('settings.tools.propertyName')} + className="border-silver dark:border-silver/40 flex w-full min-w-[130.5px] items-start rounded-lg border bg-transparent px-2 py-1 text-sm outline-hidden" + /> + + + + + + + + ) : ( + + + + + + + )} + + ); + }; + return (
@@ -1115,17 +1671,11 @@ function APIActionTable({ {t('settings.tools.name')} - {t('settings.tools.type')} - - - {t('settings.tools.filledByLLM')} + {t('settings.tools.value')} {t('settings.tools.description')} - - {t('settings.tools.value')} - - {renderPropertiesTable('headers')} + {renderHeadersTable()}
diff --git a/frontend/src/settings/types/index.ts b/frontend/src/settings/types/index.ts index 5ce3733f..c202f443 100644 --- a/frontend/src/settings/types/index.ts +++ b/frontend/src/settings/types/index.ts @@ -33,6 +33,7 @@ export type ParameterGroupType = { description: string; value: string | number; filled_by_llm: boolean; + required?: boolean; }; }; }; @@ -57,6 +58,7 @@ export type UserToolType = { description: string; filled_by_llm: boolean; value: string; + required?: boolean; }; }; additionalProperties: boolean; @@ -71,11 +73,24 @@ export type APIActionType = { name: string; url: string; description: string; - method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; + method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS'; query_params: ParameterGroupType; headers: ParameterGroupType; body: ParameterGroupType; active: boolean; + body_content_type?: + | 'application/json' + | 'application/x-www-form-urlencoded' + | 'multipart/form-data' + | 'text/plain' + | 'application/xml' + | 'application/octet-stream'; + body_encoding_rules?: { + [key: string]: { + style?: 'form' | 'spaceDelimited' | 'pipeDelimited' | 'deepObject'; + explode?: boolean; + }; + }; }; export type APIToolType = { diff --git a/tests/agents/test_base_agent.py b/tests/agents/test_base_agent.py index 1bf5aec0..7510c563 100644 --- a/tests/agents/test_base_agent.py +++ b/tests/agents/test_base_agent.py @@ -229,8 +229,14 @@ class TestBaseAgentTools: "type": "string", "description": "Test param", "filled_by_llm": True, + "required": True, + }, + "param2": { + "type": "number", + "filled_by_llm": False, + "value": 42, + "required": False, }, - "param2": {"type": "number", "filled_by_llm": False, "value": 42}, } } } From ccd29b7d4ea825c37e1a868a7e5ade93686e3d50 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 23 Dec 2025 16:33:51 +0000 Subject: [PATCH 33/93] feat: implement Docling parsers (#2202) * feat: implement Docling parsers * fix office * docling-ocr-fix * Docling smart ocr * ruff fix --------- Co-authored-by: Pavel --- application/Dockerfile | 6 +- application/api/user/sources/upload.py | 7 +- application/core/settings.py | 1 + application/parser/embedding_pipeline.py | 4 + application/parser/file/bulk.py | 113 ++++++-- application/parser/file/docling_parser.py | 330 ++++++++++++++++++++++ application/requirements.txt | 2 + 7 files changed, 439 insertions(+), 24 deletions(-) create mode 100644 application/parser/file/docling_parser.py diff --git a/application/Dockerfile b/application/Dockerfile index e33721a2..31904c5a 100644 --- a/application/Dockerfile +++ b/application/Dockerfile @@ -48,7 +48,11 @@ FROM ubuntu:24.04 as final RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository ppa:deadsnakes/ppa && \ - apt-get update && apt-get install -y --no-install-recommends python3.12 && \ + apt-get update && apt-get install -y --no-install-recommends \ + python3.12 \ + libgl1 \ + libglib2.0-0 \ + && \ ln -s /usr/bin/python3.12 /usr/bin/python && \ rm -rf /var/lib/apt/lists/* diff --git a/application/api/user/sources/upload.py b/application/api/user/sources/upload.py index 7519f7b5..6c163da4 100644 --- a/application/api/user/sources/upload.py +++ b/application/api/user/sources/upload.py @@ -76,7 +76,12 @@ class UploadFile(Resource): temp_file_path = os.path.join(temp_dir, safe_file) file.save(temp_file_path) - if zipfile.is_zipfile(temp_file_path): + # Only extract actual .zip files, not Office formats (.docx, .xlsx, .pptx) + # which are technically zip archives but should be processed as-is + is_office_format = safe_file.lower().endswith( + (".docx", ".xlsx", ".pptx", ".odt", ".ods", ".odp", ".epub") + ) + if zipfile.is_zipfile(temp_file_path) and not is_office_format: try: with zipfile.ZipFile(temp_file_path, "r") as zip_ref: zip_ref.extractall(path=temp_dir) diff --git a/application/core/settings.py b/application/core/settings.py index 0f49d8fe..d8bb6ff1 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -42,6 +42,7 @@ class Settings(BaseSettings): UPLOAD_FOLDER: str = "inputs" PARSE_PDF_AS_IMAGE: bool = False PARSE_IMAGE_REMOTE: bool = False + DOCLING_OCR_ENABLED: bool = True # Enable OCR for docling parsers (PDF, images) VECTOR_STORE: str = ( "faiss" # "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb" or "pgvector" ) diff --git a/application/parser/embedding_pipeline.py b/application/parser/embedding_pipeline.py index a777b469..80bf2786 100755 --- a/application/parser/embedding_pipeline.py +++ b/application/parser/embedding_pipeline.py @@ -65,6 +65,10 @@ def embed_and_store_documents(docs: List[Any], folder_name: str, source_id: str, if not os.path.exists(folder_name): os.makedirs(folder_name) + # Validate docs is not empty + if not docs: + raise ValueError("No documents to embed - check file format and extension") + # Initialize vector store if settings.VECTOR_STORE == "faiss": docs_init = [docs.pop(0)] diff --git a/application/parser/file/bulk.py b/application/parser/file/bulk.py index c8f2234a..dc12a4dd 100644 --- a/application/parser/file/bulk.py +++ b/application/parser/file/bulk.py @@ -10,29 +10,94 @@ from application.parser.file.epub_parser import EpubParser from application.parser.file.html_parser import HTMLParser from application.parser.file.markdown_parser import MarkdownParser from application.parser.file.rst_parser import RstParser -from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser +from application.parser.file.tabular_parser import PandasCSVParser, ExcelParser from application.parser.file.json_parser import JSONParser from application.parser.file.pptx_parser import PPTXParser from application.parser.file.image_parser import ImageParser from application.parser.schema.base import Document from application.utils import num_tokens_from_string +from application.core.settings import settings -DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = { - ".pdf": PDFParser(), - ".docx": DocxParser(), - ".csv": PandasCSVParser(), - ".xlsx":ExcelParser(), - ".epub": EpubParser(), - ".md": MarkdownParser(), - ".rst": RstParser(), - ".html": HTMLParser(), - ".mdx": MarkdownParser(), - ".json":JSONParser(), - ".pptx":PPTXParser(), - ".png": ImageParser(), - ".jpg": ImageParser(), - ".jpeg": ImageParser(), -} + +def get_default_file_extractor() -> Dict[str, BaseParser]: + """Get the default file extractor. + + Uses docling parsers by default for advanced document processing. + Falls back to standard parsers if docling is not installed. + """ + try: + from application.parser.file.docling_parser import ( + DoclingPDFParser, + DoclingDocxParser, + DoclingPPTXParser, + DoclingXLSXParser, + DoclingHTMLParser, + DoclingImageParser, + DoclingCSVParser, + DoclingAsciiDocParser, + DoclingVTTParser, + DoclingXMLParser, + ) + ocr_enabled = settings.DOCLING_OCR_ENABLED + return { + # Documents + ".pdf": DoclingPDFParser(ocr_enabled=ocr_enabled), + ".docx": DoclingDocxParser(), + ".pptx": DoclingPPTXParser(), + ".xlsx": DoclingXLSXParser(), + # Web formats + ".html": DoclingHTMLParser(), + ".xhtml": DoclingHTMLParser(), + # Data formats + ".csv": DoclingCSVParser(), + ".json": JSONParser(), # Keep JSON parser (specialized handling) + # Text/markup formats + ".md": MarkdownParser(), # Keep markdown parser (specialized handling) + ".mdx": MarkdownParser(), + ".rst": RstParser(), + ".adoc": DoclingAsciiDocParser(), + ".asciidoc": DoclingAsciiDocParser(), + # Images (with OCR) + ".png": DoclingImageParser(ocr_enabled=ocr_enabled), + ".jpg": DoclingImageParser(ocr_enabled=ocr_enabled), + ".jpeg": DoclingImageParser(ocr_enabled=ocr_enabled), + ".tiff": DoclingImageParser(ocr_enabled=ocr_enabled), + ".tif": DoclingImageParser(ocr_enabled=ocr_enabled), + ".bmp": DoclingImageParser(ocr_enabled=ocr_enabled), + ".webp": DoclingImageParser(ocr_enabled=ocr_enabled), + # Media/subtitles + ".vtt": DoclingVTTParser(), + # Specialized XML formats + ".xml": DoclingXMLParser(), + # Formats docling doesn't support - use standard parsers + ".epub": EpubParser(), + } + except ImportError: + logging.warning( + "docling is not installed. Using standard parsers. " + "For advanced document parsing, install with: pip install docling" + ) + # Fallback to standard parsers + return { + ".pdf": PDFParser(), + ".docx": DocxParser(), + ".csv": PandasCSVParser(), + ".xlsx": ExcelParser(), + ".epub": EpubParser(), + ".md": MarkdownParser(), + ".rst": RstParser(), + ".html": HTMLParser(), + ".mdx": MarkdownParser(), + ".json": JSONParser(), + ".pptx": PPTXParser(), + ".png": ImageParser(), + ".jpg": ImageParser(), + ".jpeg": ImageParser(), + } + + +# For backwards compatibility +DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = get_default_file_extractor() class SimpleDirectoryReader(BaseReader): @@ -83,7 +148,10 @@ class SimpleDirectoryReader(BaseReader): self.recursive = recursive self.exclude_hidden = exclude_hidden - self.required_exts = required_exts + # Normalize extensions to lowercase for case-insensitive matching + self.required_exts = ( + [ext.lower() for ext in required_exts] if required_exts else None + ) self.num_files_limit = num_files_limit if input_files: @@ -112,7 +180,7 @@ class SimpleDirectoryReader(BaseReader): continue elif ( self.required_exts is not None - and input_file.suffix not in self.required_exts + and input_file.suffix.lower() not in self.required_exts ): continue else: @@ -149,8 +217,9 @@ class SimpleDirectoryReader(BaseReader): self.file_token_counts = {} for input_file in self.input_files: - if input_file.suffix in self.file_extractor: - parser = self.file_extractor[input_file.suffix] + suffix_lower = input_file.suffix.lower() + if suffix_lower in self.file_extractor: + parser = self.file_extractor[suffix_lower] if not parser.parser_config_set: parser.init_parser() data = parser.parse_file(input_file, errors=self.errors) @@ -232,7 +301,7 @@ class SimpleDirectoryReader(BaseReader): if subtree: result[item.name] = subtree else: - if self.required_exts is not None and item.suffix not in self.required_exts: + if self.required_exts is not None and item.suffix.lower() not in self.required_exts: continue full_path = str(item.resolve()) diff --git a/application/parser/file/docling_parser.py b/application/parser/file/docling_parser.py new file mode 100644 index 00000000..a12431d1 --- /dev/null +++ b/application/parser/file/docling_parser.py @@ -0,0 +1,330 @@ +"""Docling parser. + +Uses docling library for advanced document parsing with layout detection, +table structure recognition, and unified document representation. + +Supports: PDF, DOCX, PPTX, XLSX, HTML, XHTML, CSV, Markdown, AsciiDoc, +images (PNG, JPEG, TIFF, BMP, WEBP), WebVTT, and specialized XML formats. +""" +import importlib.util +import logging +from pathlib import Path +from typing import Dict, List, Optional, Union + +from application.parser.file.base_parser import BaseParser + +logger = logging.getLogger(__name__) + + +class DoclingParser(BaseParser): + """Parser using docling for advanced document processing. + + Docling provides: + - Advanced PDF layout analysis + - Table structure recognition + - Reading order detection + - OCR for scanned documents (supports RapidOCR) + - Unified DoclingDocument format + - Export to Markdown + + Uses hybrid OCR approach by default: + - Text regions: Direct PDF text extraction (fast) + - Bitmap/image regions: OCR only these areas (smart) + """ + + def __init__( + self, + ocr_enabled: bool = True, + table_structure: bool = True, + export_format: str = "markdown", + use_rapidocr: bool = True, + ocr_languages: Optional[List[str]] = None, + force_full_page_ocr: bool = False, + ): + """Initialize DoclingParser. + + Args: + ocr_enabled: Enable OCR for bitmap/image regions in documents + table_structure: Enable table structure recognition + export_format: Output format ('markdown', 'text', 'html') + use_rapidocr: Use RapidOCR engine (default True, works well in Docker) + ocr_languages: List of OCR languages (default: ['english']) + force_full_page_ocr: Force OCR on entire page (False = smart hybrid OCR) + """ + super().__init__() + self.ocr_enabled = ocr_enabled + self.table_structure = table_structure + self.export_format = export_format + self.use_rapidocr = use_rapidocr + self.ocr_languages = ocr_languages or ["english"] + self.force_full_page_ocr = force_full_page_ocr + self._converter = None + + def _create_converter(self): + """Create a docling converter with hybrid OCR configuration. + + Uses smart OCR approach: + - When ocr_enabled=True and force_full_page_ocr=False (default): + Layout model detects text vs bitmap regions, OCR only runs on bitmaps + - When ocr_enabled=True and force_full_page_ocr=True: + OCR runs on entire page (for scanned documents/images) + - When ocr_enabled=False: + No OCR, only native text extraction + + Returns: + DocumentConverter instance + """ + from docling.document_converter import ( + DocumentConverter, + ImageFormatOption, + InputFormat, + PdfFormatOption, + ) + from docling.datamodel.pipeline_options import PdfPipelineOptions + + pipeline_options = PdfPipelineOptions( + do_ocr=self.ocr_enabled, + do_table_structure=self.table_structure, + ) + + if self.ocr_enabled: + ocr_options = self._get_ocr_options() + if ocr_options is not None: + pipeline_options.ocr_options = ocr_options + + return DocumentConverter( + format_options={ + InputFormat.PDF: PdfFormatOption( + pipeline_options=pipeline_options, + ), + InputFormat.IMAGE: ImageFormatOption( + pipeline_options=pipeline_options, + ), + } + ) + + def _init_parser(self) -> Dict: + """Initialize the docling converter with hybrid OCR.""" + logger.info("Initializing DoclingParser...") + logger.info(f" ocr_enabled={self.ocr_enabled}") + logger.info(f" force_full_page_ocr={self.force_full_page_ocr}") + logger.info(f" use_rapidocr={self.use_rapidocr}") + + if importlib.util.find_spec("docling.document_converter") is None: + raise ImportError( + "docling is required for DoclingParser. " + "Install it with: pip install docling" + ) + + # Create converter with hybrid OCR (smart: text direct, bitmaps OCR'd) + self._converter = self._create_converter() + + logger.info("DoclingParser initialized successfully") + return { + "ocr_enabled": self.ocr_enabled, + "table_structure": self.table_structure, + "export_format": self.export_format, + "use_rapidocr": self.use_rapidocr, + "ocr_languages": self.ocr_languages, + "force_full_page_ocr": self.force_full_page_ocr, + } + + def _get_ocr_options(self): + """Get OCR options based on configuration. + + Returns RapidOcrOptions if use_rapidocr is True and available, + otherwise returns None to use docling defaults. + """ + if not self.use_rapidocr: + return None + + try: + from docling.datamodel.pipeline_options import RapidOcrOptions + + return RapidOcrOptions( + lang=self.ocr_languages, + force_full_page_ocr=self.force_full_page_ocr, + ) + except ImportError as e: + logger.warning(f"Failed to import RapidOcrOptions: {e}") + return None + except Exception as e: + logger.error(f"Error creating RapidOcrOptions: {e}") + return None + + def _export_content(self, document) -> str: + """Export document content in the configured format. + + Handles edge case where text is nested under picture elements (e.g., OCR'd + images). If the standard export returns minimal content but document.texts + contains extracted text, falls back to direct text extraction. + """ + if self.export_format == "markdown": + content = document.export_to_markdown() + elif self.export_format == "html": + content = document.export_to_html() + else: + content = document.export_to_text() + + # Handle case where text is nested under pictures (common with OCR'd images) + # Standard exports may return just "" while actual text exists + stripped_content = content.strip() + is_minimal = len(stripped_content) < 50 or stripped_content == "" + + if is_minimal and hasattr(document, "texts") and document.texts: + # Extract text directly from document.texts + extracted_texts = [t.text for t in document.texts if t.text] + if extracted_texts: + logger.info( + f"Standard export minimal ({len(stripped_content)} chars), " + f"extracting {len(extracted_texts)} texts directly" + ) + return "\n\n".join(extracted_texts) + + return content + + def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]: + """Parse file using docling with hybrid OCR. + + Uses smart OCR approach where the layout model detects text vs bitmap + regions. Text is extracted directly, bitmaps are OCR'd only when needed. + + Args: + file: Path to the file to parse + errors: Error handling mode (ignored, docling handles internally) + + Returns: + Parsed document content as markdown string + """ + logger.info(f"parse_file called for: {file}") + + if self._converter is None: + self._init_parser() + + try: + logger.info(f"Converting file with hybrid OCR: {file}") + result = self._converter.convert(str(file)) + content = self._export_content(result.document) + logger.info(f"Parse complete, content length: {len(content)} chars") + + return content + + except Exception as e: + logger.error(f"Error parsing file with docling: {e}", exc_info=True) + if errors == "ignore": + return f"[Error parsing file with docling: {str(e)}]" + raise + + +class DoclingPDFParser(DoclingParser): + """Docling-based PDF parser with advanced features and RapidOCR support. + + Uses hybrid OCR approach by default: + - Text regions: Direct PDF text extraction (fast) + - Bitmap/image regions: OCR only these areas (smart) + + Set force_full_page_ocr=True only for fully scanned documents. + """ + + def __init__( + self, + ocr_enabled: bool = True, + table_structure: bool = True, + use_rapidocr: bool = True, + ocr_languages: Optional[List[str]] = None, + force_full_page_ocr: bool = False, + ): + super().__init__( + ocr_enabled=ocr_enabled, + table_structure=table_structure, + export_format="markdown", + use_rapidocr=use_rapidocr, + ocr_languages=ocr_languages, + force_full_page_ocr=force_full_page_ocr, + ) + + +class DoclingDocxParser(DoclingParser): + """Docling-based DOCX parser.""" + + def __init__(self): + super().__init__(export_format="markdown") + + +class DoclingPPTXParser(DoclingParser): + """Docling-based PPTX parser.""" + + def __init__(self): + super().__init__(export_format="markdown") + + +class DoclingXLSXParser(DoclingParser): + """Docling-based XLSX parser with table structure.""" + + def __init__(self): + super().__init__(table_structure=True, export_format="markdown") + + +class DoclingHTMLParser(DoclingParser): + """Docling-based HTML parser.""" + + def __init__(self): + super().__init__(export_format="markdown") + + +class DoclingImageParser(DoclingParser): + """Docling-based image parser with OCR and RapidOCR support. + + For images, force_full_page_ocr=True is used since images are entirely + visual and require full OCR to extract any text. + """ + + def __init__( + self, + ocr_enabled: bool = True, + use_rapidocr: bool = True, + ocr_languages: Optional[List[str]] = None, + force_full_page_ocr: bool = True, + ): + super().__init__( + ocr_enabled=ocr_enabled, + export_format="markdown", + use_rapidocr=use_rapidocr, + ocr_languages=ocr_languages, + force_full_page_ocr=force_full_page_ocr, + ) + + +class DoclingCSVParser(DoclingParser): + """Docling-based CSV parser.""" + + def __init__(self): + super().__init__(table_structure=True, export_format="markdown") + + +class DoclingMarkdownParser(DoclingParser): + """Docling-based Markdown parser.""" + + def __init__(self): + super().__init__(export_format="markdown") + + +class DoclingAsciiDocParser(DoclingParser): + """Docling-based AsciiDoc parser.""" + + def __init__(self): + super().__init__(export_format="markdown") + + +class DoclingVTTParser(DoclingParser): + """Docling-based WebVTT (video text tracks) parser.""" + + def __init__(self): + super().__init__(export_format="markdown") + + +class DoclingXMLParser(DoclingParser): + """Docling-based XML parser (USPTO, JATS).""" + + def __init__(self): + super().__init__(export_format="markdown") diff --git a/application/requirements.txt b/application/requirements.txt index c88cabaf..89e8e042 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -5,6 +5,8 @@ celery==5.6.0 cryptography==46.0.3 dataclasses-json==0.6.7 docling>=2.16.0 +rapidocr>=1.4.0 +onnxruntime>=1.19.0 docx2txt==0.8 duckduckgo-search==8.1.1 ebooklib==0.20 From 83e7a928f1e05f28756f9689c0386bb06dc0c88f Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 23 Dec 2025 23:36:15 +0000 Subject: [PATCH 34/93] bump deps --- application/requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/application/requirements.txt b/application/requirements.txt index 89e8e042..88402ae7 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -7,7 +7,7 @@ dataclasses-json==0.6.7 docling>=2.16.0 rapidocr>=1.4.0 onnxruntime>=1.19.0 -docx2txt==0.8 +docx2txt==0.9 duckduckgo-search==8.1.1 ebooklib==0.20 escodegen==1.0.11 @@ -37,14 +37,14 @@ langchain-community==0.4.1 langchain-core==1.2.4 langchain-openai==1.1.1 langchain-text-splitters==1.0.0 -langsmith==0.4.58 +langsmith==0.5.0 lazy-object-proxy==1.12.0 lxml==6.0.2 -markupsafe==3.0.2 -marshmallow==3.26.1 +markupsafe==3.0.3 +marshmallow==3.26.2 mpmath==1.3.0 multidict==6.7.0 -mypy-extensions==1.0.0 +mypy-extensions==1.1.0 networkx==3.6.1 numpy==2.2.1 openai==2.9.0 @@ -75,7 +75,7 @@ referencing>=0.28.0,<0.31.0 regex==2025.11.3 requests==2.32.5 retry==0.9.2 -sentence-transformers==5.1.2 +sentence-transformers==5.2.0 tiktoken==0.12.0 tokenizers==0.22.1 torch==2.9.1 From 98e949d2fd9522a91b430ac239c5d945304601b1 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 24 Dec 2025 15:05:35 +0000 Subject: [PATCH 35/93] Patches (#2218) * feat: implement URL validation to prevent SSRF * feat: add zip extraction security * ruff fixes --- application/agents/tools/api_tool.py | 24 ++ application/agents/tools/read_webpage.py | 13 +- application/core/url_validation.py | 181 +++++++++++ application/parser/remote/crawler_loader.py | 19 +- application/parser/remote/crawler_markdown.py | 15 +- application/parser/remote/sitemap_loader.py | 17 +- application/worker.py | 120 ++++++- tests/core/test_url_validation.py | 197 ++++++++++++ tests/parser/remote/test_crawler_loader.py | 41 ++- tests/parser/remote/test_crawler_markdown.py | 36 ++- tests/test_zip_extraction_security.py | 293 ++++++++++++++++++ 11 files changed, 929 insertions(+), 27 deletions(-) create mode 100644 application/core/url_validation.py create mode 100644 tests/core/test_url_validation.py create mode 100644 tests/test_zip_extraction_security.py diff --git a/application/agents/tools/api_tool.py b/application/agents/tools/api_tool.py index 6bd2eb8d..e010b51b 100644 --- a/application/agents/tools/api_tool.py +++ b/application/agents/tools/api_tool.py @@ -11,6 +11,7 @@ from application.agents.tools.api_body_serializer import ( RequestBodySerializer, ) from application.agents.tools.base import Tool +from application.core.url_validation import validate_url, SSRFError logger = logging.getLogger(__name__) @@ -73,6 +74,17 @@ class APITool(Tool): request_headers = headers.copy() if headers else {} response = None + # Validate URL to prevent SSRF attacks + try: + validate_url(request_url) + except SSRFError as e: + logger.error(f"URL validation failed: {e}") + return { + "status_code": None, + "message": f"URL validation error: {e}", + "data": None, + } + try: path_params_used = set() if query_params: @@ -90,6 +102,18 @@ class APITool(Tool): query_string = urlencode(remaining_params) separator = "&" if "?" in request_url else "?" request_url = f"{request_url}{separator}{query_string}" + + # Re-validate URL after parameter substitution to prevent SSRF via path params + try: + validate_url(request_url) + except SSRFError as e: + logger.error(f"URL validation failed after parameter substitution: {e}") + return { + "status_code": None, + "message": f"URL validation error: {e}", + "data": None, + } + # Serialize body based on content type if body and body != {}: diff --git a/application/agents/tools/read_webpage.py b/application/agents/tools/read_webpage.py index e87c79e3..f0321a5a 100644 --- a/application/agents/tools/read_webpage.py +++ b/application/agents/tools/read_webpage.py @@ -1,7 +1,7 @@ import requests from markdownify import markdownify from application.agents.tools.base import Tool -from urllib.parse import urlparse +from application.core.url_validation import validate_url, SSRFError class ReadWebpageTool(Tool): """ @@ -31,11 +31,12 @@ class ReadWebpageTool(Tool): if not url: return "Error: URL parameter is missing." - # Ensure the URL has a scheme (if not, default to http) - parsed_url = urlparse(url) - if not parsed_url.scheme: - url = "http://" + url - + # Validate URL to prevent SSRF attacks + try: + url = validate_url(url) + except SSRFError as e: + return f"Error: URL validation failed - {e}" + try: response = requests.get(url, timeout=10, headers={'User-Agent': 'DocsGPT-Agent/1.0'}) response.raise_for_status() # Raise an exception for HTTP errors (4xx or 5xx) diff --git a/application/core/url_validation.py b/application/core/url_validation.py new file mode 100644 index 00000000..acd8a523 --- /dev/null +++ b/application/core/url_validation.py @@ -0,0 +1,181 @@ +""" +URL validation utilities to prevent SSRF (Server-Side Request Forgery) attacks. + +This module provides functions to validate URLs before making HTTP requests, +blocking access to internal networks, cloud metadata services, and other +potentially dangerous endpoints. +""" + +import ipaddress +import socket +from urllib.parse import urlparse +from typing import Optional, Set + + +class SSRFError(Exception): + """Raised when a URL fails SSRF validation.""" + pass + + +# Blocked hostnames that should never be accessed +BLOCKED_HOSTNAMES: Set[str] = { + "localhost", + "localhost.localdomain", + "metadata.google.internal", + "metadata", +} + +# Cloud metadata IP addresses (AWS, GCP, Azure, etc.) +METADATA_IPS: Set[str] = { + "169.254.169.254", # AWS, GCP, Azure metadata + "169.254.170.2", # AWS ECS task metadata + "fd00:ec2::254", # AWS IPv6 metadata +} + +# Allowed schemes for external requests +ALLOWED_SCHEMES: Set[str] = {"http", "https"} + + +def is_private_ip(ip_str: str) -> bool: + """ + Check if an IP address is private, loopback, or link-local. + + Args: + ip_str: IP address as a string + + Returns: + True if the IP is private/internal, False otherwise + """ + try: + ip = ipaddress.ip_address(ip_str) + return ( + ip.is_private or + ip.is_loopback or + ip.is_link_local or + ip.is_reserved or + ip.is_multicast or + ip.is_unspecified + ) + except ValueError: + # If we can't parse it as an IP, return False + return False + + +def is_metadata_ip(ip_str: str) -> bool: + """ + Check if an IP address is a cloud metadata service IP. + + Args: + ip_str: IP address as a string + + Returns: + True if the IP is a metadata service, False otherwise + """ + return ip_str in METADATA_IPS + + +def resolve_hostname(hostname: str) -> Optional[str]: + """ + Resolve a hostname to an IP address. + + Args: + hostname: The hostname to resolve + + Returns: + The resolved IP address, or None if resolution fails + """ + try: + return socket.gethostbyname(hostname) + except socket.gaierror: + return None + + +def validate_url(url: str, allow_localhost: bool = False) -> str: + """ + Validate a URL to prevent SSRF attacks. + + This function checks that: + 1. The URL has an allowed scheme (http or https) + 2. The hostname is not a blocked hostname + 3. The resolved IP is not a private/internal IP + 4. The resolved IP is not a cloud metadata service + + Args: + url: The URL to validate + allow_localhost: If True, allow localhost connections (for testing only) + + Returns: + The validated URL (with scheme added if missing) + + Raises: + SSRFError: If the URL fails validation + """ + # Ensure URL has a scheme + if not urlparse(url).scheme: + url = "http://" + url + + parsed = urlparse(url) + + # Check scheme + if parsed.scheme not in ALLOWED_SCHEMES: + raise SSRFError(f"URL scheme '{parsed.scheme}' is not allowed. Only HTTP(S) is permitted.") + + hostname = parsed.hostname + if not hostname: + raise SSRFError("URL must have a valid hostname.") + + hostname_lower = hostname.lower() + + # Check blocked hostnames + if hostname_lower in BLOCKED_HOSTNAMES and not allow_localhost: + raise SSRFError(f"Access to '{hostname}' is not allowed.") + + # Check if hostname is an IP address directly + try: + ip = ipaddress.ip_address(hostname) + ip_str = str(ip) + + if is_metadata_ip(ip_str): + raise SSRFError("Access to cloud metadata services is not allowed.") + + if is_private_ip(ip_str) and not allow_localhost: + raise SSRFError("Access to private/internal IP addresses is not allowed.") + + return url + except ValueError: + # Not an IP address, it's a hostname - resolve it + pass + + # Resolve hostname and check the IP + resolved_ip = resolve_hostname(hostname) + if resolved_ip is None: + raise SSRFError(f"Unable to resolve hostname: {hostname}") + + if is_metadata_ip(resolved_ip): + raise SSRFError("Access to cloud metadata services is not allowed.") + + if is_private_ip(resolved_ip) and not allow_localhost: + raise SSRFError("Access to private/internal networks is not allowed.") + + return url + + +def validate_url_safe(url: str, allow_localhost: bool = False) -> tuple[bool, str, Optional[str]]: + """ + Validate a URL and return a tuple with validation result. + + This is a non-throwing version of validate_url for cases where + you want to handle validation failures gracefully. + + Args: + url: The URL to validate + allow_localhost: If True, allow localhost connections (for testing only) + + Returns: + Tuple of (is_valid, validated_url_or_original, error_message_or_none) + """ + try: + validated = validate_url(url, allow_localhost) + return (True, validated, None) + except SSRFError as e: + return (False, url, str(e)) diff --git a/application/parser/remote/crawler_loader.py b/application/parser/remote/crawler_loader.py index 2ff6cf6f..1bfd2276 100644 --- a/application/parser/remote/crawler_loader.py +++ b/application/parser/remote/crawler_loader.py @@ -4,6 +4,7 @@ from urllib.parse import urlparse, urljoin from bs4 import BeautifulSoup from application.parser.remote.base import BaseRemote from application.parser.schema.base import Document +from application.core.url_validation import validate_url, SSRFError from langchain_community.document_loaders import WebBaseLoader class CrawlerLoader(BaseRemote): @@ -16,9 +17,12 @@ class CrawlerLoader(BaseRemote): if isinstance(url, list) and url: url = url[0] - # Check if the URL scheme is provided, if not, assume http - if not urlparse(url).scheme: - url = "http://" + url + # Validate URL to prevent SSRF attacks + try: + url = validate_url(url) + except SSRFError as e: + logging.error(f"URL validation failed: {e}") + return [] visited_urls = set() base_url = urlparse(url).scheme + "://" + urlparse(url).hostname @@ -30,7 +34,14 @@ class CrawlerLoader(BaseRemote): visited_urls.add(current_url) try: - response = requests.get(current_url) + # Validate each URL before making requests + try: + validate_url(current_url) + except SSRFError as e: + logging.warning(f"Skipping URL due to validation failure: {current_url} - {e}") + continue + + response = requests.get(current_url, timeout=30) response.raise_for_status() loader = self.loader([current_url]) docs = loader.load() diff --git a/application/parser/remote/crawler_markdown.py b/application/parser/remote/crawler_markdown.py index 3d199332..8fc4c92c 100644 --- a/application/parser/remote/crawler_markdown.py +++ b/application/parser/remote/crawler_markdown.py @@ -2,6 +2,7 @@ import requests from urllib.parse import urlparse, urljoin from bs4 import BeautifulSoup from application.parser.remote.base import BaseRemote +from application.core.url_validation import validate_url, SSRFError import re from markdownify import markdownify from application.parser.schema.base import Document @@ -25,9 +26,12 @@ class CrawlerLoader(BaseRemote): if isinstance(url, list) and url: url = url[0] - # Ensure the URL has a scheme (if not, default to http) - if not urlparse(url).scheme: - url = "http://" + url + # Validate URL to prevent SSRF attacks + try: + url = validate_url(url) + except SSRFError as e: + print(f"URL validation failed: {e}") + return [] # Keep track of visited URLs to avoid revisiting the same page visited_urls = set() @@ -78,9 +82,14 @@ class CrawlerLoader(BaseRemote): def _fetch_page(self, url): try: + # Validate URL before fetching to prevent SSRF + validate_url(url) response = self.session.get(url, timeout=10) response.raise_for_status() return response.text + except SSRFError as e: + print(f"URL validation failed for {url}: {e}") + return None except requests.exceptions.RequestException as e: print(f"Error fetching URL {url}: {e}") return None diff --git a/application/parser/remote/sitemap_loader.py b/application/parser/remote/sitemap_loader.py index 6d54ea9b..ff7c1ede 100644 --- a/application/parser/remote/sitemap_loader.py +++ b/application/parser/remote/sitemap_loader.py @@ -3,6 +3,7 @@ import requests import re # Import regular expression library import xml.etree.ElementTree as ET from application.parser.remote.base import BaseRemote +from application.core.url_validation import validate_url, SSRFError class SitemapLoader(BaseRemote): def __init__(self, limit=20): @@ -14,7 +15,14 @@ class SitemapLoader(BaseRemote): sitemap_url= inputs # Check if the input is a list and if it is, use the first element if isinstance(sitemap_url, list) and sitemap_url: - url = sitemap_url[0] + sitemap_url = sitemap_url[0] + + # Validate URL to prevent SSRF attacks + try: + sitemap_url = validate_url(sitemap_url) + except SSRFError as e: + logging.error(f"URL validation failed: {e}") + return [] urls = self._extract_urls(sitemap_url) if not urls: @@ -40,8 +48,13 @@ class SitemapLoader(BaseRemote): def _extract_urls(self, sitemap_url): try: - response = requests.get(sitemap_url) + # Validate URL before fetching to prevent SSRF + validate_url(sitemap_url) + response = requests.get(sitemap_url, timeout=30) response.raise_for_status() # Raise an exception for HTTP errors + except SSRFError as e: + print(f"URL validation failed for sitemap: {sitemap_url}. Error: {e}") + return [] except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e: print(f"Failed to fetch sitemap: {sitemap_url}. Error: {e}") return [] diff --git a/application/worker.py b/application/worker.py index f45e94a5..fa2b6cd7 100755 --- a/application/worker.py +++ b/application/worker.py @@ -63,10 +63,111 @@ current_dir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ) +# Zip extraction security limits +MAX_UNCOMPRESSED_SIZE = 500 * 1024 * 1024 # 500 MB max uncompressed size +MAX_FILE_COUNT = 10000 # Maximum number of files to extract +MAX_COMPRESSION_RATIO = 100 # Maximum compression ratio (to detect zip bombs) + + +class ZipExtractionError(Exception): + """Raised when zip extraction fails due to security constraints.""" + pass + + +def _is_path_safe(base_path: str, target_path: str) -> bool: + """ + Check if target_path is safely within base_path (prevents zip slip attacks). + + Args: + base_path: The base directory where extraction should occur. + target_path: The full path where a file would be extracted. + + Returns: + True if the path is safe, False otherwise. + """ + # Resolve to absolute paths and check containment + base_resolved = os.path.realpath(base_path) + target_resolved = os.path.realpath(target_path) + return target_resolved.startswith(base_resolved + os.sep) or target_resolved == base_resolved + + +def _validate_zip_safety(zip_path: str, extract_to: str) -> None: + """ + Validate a zip file for security issues before extraction. + + Checks for: + - Zip bombs (excessive compression ratio or uncompressed size) + - Too many files + - Path traversal attacks (zip slip) + + Args: + zip_path: Path to the zip file. + extract_to: Destination directory. + + Raises: + ZipExtractionError: If the zip file fails security validation. + """ + try: + with zipfile.ZipFile(zip_path, "r") as zip_ref: + # Get compressed size + compressed_size = os.path.getsize(zip_path) + + # Calculate total uncompressed size and file count + total_uncompressed = 0 + file_count = 0 + + for info in zip_ref.infolist(): + file_count += 1 + + # Check file count limit + if file_count > MAX_FILE_COUNT: + raise ZipExtractionError( + f"Zip file contains too many files (>{MAX_FILE_COUNT}). " + "This may be a zip bomb attack." + ) + + # Accumulate uncompressed size + total_uncompressed += info.file_size + + # Check total uncompressed size + if total_uncompressed > MAX_UNCOMPRESSED_SIZE: + raise ZipExtractionError( + f"Zip file uncompressed size exceeds limit " + f"({total_uncompressed / (1024*1024):.1f} MB > " + f"{MAX_UNCOMPRESSED_SIZE / (1024*1024):.1f} MB). " + "This may be a zip bomb attack." + ) + + # Check for path traversal (zip slip) + target_path = os.path.join(extract_to, info.filename) + if not _is_path_safe(extract_to, target_path): + raise ZipExtractionError( + f"Zip file contains path traversal attempt: {info.filename}" + ) + + # Check compression ratio (only if compressed size is meaningful) + if compressed_size > 0 and total_uncompressed > 0: + compression_ratio = total_uncompressed / compressed_size + if compression_ratio > MAX_COMPRESSION_RATIO: + raise ZipExtractionError( + f"Zip file has suspicious compression ratio ({compression_ratio:.1f}:1 > " + f"{MAX_COMPRESSION_RATIO}:1). This may be a zip bomb attack." + ) + + except zipfile.BadZipFile as e: + raise ZipExtractionError(f"Invalid or corrupted zip file: {e}") + def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5): """ - Recursively extract zip files with a limit on recursion depth. + Recursively extract zip files with security protections. + + Security measures: + - Limits recursion depth to prevent infinite loops + - Validates uncompressed size to prevent zip bombs + - Limits number of files to prevent resource exhaustion + - Checks compression ratio to detect zip bombs + - Validates paths to prevent zip slip attacks Args: zip_path (str): Path to the zip file to be extracted. @@ -77,20 +178,33 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5): if current_depth > max_depth: logging.warning(f"Reached maximum recursion depth of {max_depth}") return + try: + # Validate zip file safety before extraction + _validate_zip_safety(zip_path, extract_to) + + # Safe to extract with zipfile.ZipFile(zip_path, "r") as zip_ref: zip_ref.extractall(extract_to) os.remove(zip_path) # Remove the zip file after extracting + + except ZipExtractionError as e: + logging.error(f"Zip security validation failed for {zip_path}: {e}") + # Remove the potentially malicious zip file + try: + os.remove(zip_path) + except OSError: + pass + return except Exception as e: logging.error(f"Error extracting zip file {zip_path}: {e}", exc_info=True) return - # Check for nested zip files and extract them + # Check for nested zip files and extract them for root, dirs, files in os.walk(extract_to): for file in files: if file.endswith(".zip"): # If a nested zip file is found, extract it recursively - file_path = os.path.join(root, file) extract_zip_recursive(file_path, root, current_depth + 1, max_depth) diff --git a/tests/core/test_url_validation.py b/tests/core/test_url_validation.py new file mode 100644 index 00000000..924e5cde --- /dev/null +++ b/tests/core/test_url_validation.py @@ -0,0 +1,197 @@ +"""Tests for SSRF URL validation module.""" + +import pytest +from unittest.mock import patch + +from application.core.url_validation import ( + SSRFError, + validate_url, + validate_url_safe, + is_private_ip, + is_metadata_ip, +) + + +class TestIsPrivateIP: + """Tests for is_private_ip function.""" + + def test_loopback_ipv4(self): + assert is_private_ip("127.0.0.1") is True + assert is_private_ip("127.255.255.255") is True + + def test_private_class_a(self): + assert is_private_ip("10.0.0.1") is True + assert is_private_ip("10.255.255.255") is True + + def test_private_class_b(self): + assert is_private_ip("172.16.0.1") is True + assert is_private_ip("172.31.255.255") is True + + def test_private_class_c(self): + assert is_private_ip("192.168.0.1") is True + assert is_private_ip("192.168.255.255") is True + + def test_link_local(self): + assert is_private_ip("169.254.0.1") is True + + def test_public_ip(self): + assert is_private_ip("8.8.8.8") is False + assert is_private_ip("1.1.1.1") is False + assert is_private_ip("93.184.216.34") is False + + def test_invalid_ip(self): + assert is_private_ip("not-an-ip") is False + assert is_private_ip("") is False + + +class TestIsMetadataIP: + """Tests for is_metadata_ip function.""" + + def test_aws_metadata_ip(self): + assert is_metadata_ip("169.254.169.254") is True + + def test_aws_ecs_metadata_ip(self): + assert is_metadata_ip("169.254.170.2") is True + + def test_non_metadata_ip(self): + assert is_metadata_ip("8.8.8.8") is False + assert is_metadata_ip("10.0.0.1") is False + + +class TestValidateUrl: + """Tests for validate_url function.""" + + def test_adds_scheme_if_missing(self): + with patch("application.core.url_validation.resolve_hostname") as mock_resolve: + mock_resolve.return_value = "93.184.216.34" # Public IP + result = validate_url("example.com") + assert result == "http://example.com" + + def test_preserves_https_scheme(self): + with patch("application.core.url_validation.resolve_hostname") as mock_resolve: + mock_resolve.return_value = "93.184.216.34" + result = validate_url("https://example.com") + assert result == "https://example.com" + + def test_blocks_localhost(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://localhost") + assert "localhost" in str(exc_info.value).lower() + + def test_blocks_localhost_localdomain(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://localhost.localdomain") + assert "not allowed" in str(exc_info.value).lower() + + def test_blocks_loopback_ip(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://127.0.0.1") + assert "private" in str(exc_info.value).lower() or "internal" in str(exc_info.value).lower() + + def test_blocks_private_ip_class_a(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://10.0.0.1") + assert "private" in str(exc_info.value).lower() or "internal" in str(exc_info.value).lower() + + def test_blocks_private_ip_class_b(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://172.16.0.1") + assert "private" in str(exc_info.value).lower() or "internal" in str(exc_info.value).lower() + + def test_blocks_private_ip_class_c(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://192.168.1.1") + assert "private" in str(exc_info.value).lower() or "internal" in str(exc_info.value).lower() + + def test_blocks_aws_metadata_ip(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://169.254.169.254") + assert "metadata" in str(exc_info.value).lower() + + def test_blocks_aws_metadata_with_path(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://169.254.169.254/latest/meta-data/") + assert "metadata" in str(exc_info.value).lower() + + def test_blocks_gcp_metadata_hostname(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://metadata.google.internal") + assert "not allowed" in str(exc_info.value).lower() + + def test_blocks_ftp_scheme(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("ftp://example.com") + assert "scheme" in str(exc_info.value).lower() + + def test_blocks_file_scheme(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("file:///etc/passwd") + assert "scheme" in str(exc_info.value).lower() + + def test_blocks_hostname_resolving_to_private_ip(self): + with patch("application.core.url_validation.resolve_hostname") as mock_resolve: + mock_resolve.return_value = "192.168.1.1" + with pytest.raises(SSRFError) as exc_info: + validate_url("http://internal.example.com") + assert "private" in str(exc_info.value).lower() or "internal" in str(exc_info.value).lower() + + def test_blocks_hostname_resolving_to_metadata_ip(self): + with patch("application.core.url_validation.resolve_hostname") as mock_resolve: + mock_resolve.return_value = "169.254.169.254" + with pytest.raises(SSRFError) as exc_info: + validate_url("http://evil.example.com") + assert "metadata" in str(exc_info.value).lower() + + def test_allows_public_ip(self): + result = validate_url("http://8.8.8.8") + assert result == "http://8.8.8.8" + + def test_allows_public_hostname(self): + with patch("application.core.url_validation.resolve_hostname") as mock_resolve: + mock_resolve.return_value = "93.184.216.34" + result = validate_url("https://example.com") + assert result == "https://example.com" + + def test_raises_on_unresolvable_hostname(self): + with patch("application.core.url_validation.resolve_hostname") as mock_resolve: + mock_resolve.return_value = None + with pytest.raises(SSRFError) as exc_info: + validate_url("http://nonexistent.invalid") + assert "resolve" in str(exc_info.value).lower() + + def test_raises_on_empty_hostname(self): + with pytest.raises(SSRFError) as exc_info: + validate_url("http://") + assert "hostname" in str(exc_info.value).lower() + + def test_allow_localhost_flag(self): + # Should work with allow_localhost=True + result = validate_url("http://localhost", allow_localhost=True) + assert result == "http://localhost" + + result = validate_url("http://127.0.0.1", allow_localhost=True) + assert result == "http://127.0.0.1" + + +class TestValidateUrlSafe: + """Tests for validate_url_safe non-throwing function.""" + + def test_returns_tuple_on_success(self): + with patch("application.core.url_validation.resolve_hostname") as mock_resolve: + mock_resolve.return_value = "93.184.216.34" + is_valid, url, error = validate_url_safe("https://example.com") + assert is_valid is True + assert url == "https://example.com" + assert error is None + + def test_returns_tuple_on_failure(self): + is_valid, url, error = validate_url_safe("http://localhost") + assert is_valid is False + assert url == "http://localhost" + assert error is not None + assert "localhost" in error.lower() + + def test_returns_error_message_for_private_ip(self): + is_valid, url, error = validate_url_safe("http://192.168.1.1") + assert is_valid is False + assert "private" in error.lower() or "internal" in error.lower() diff --git a/tests/parser/remote/test_crawler_loader.py b/tests/parser/remote/test_crawler_loader.py index 92ffdc84..06d27517 100644 --- a/tests/parser/remote/test_crawler_loader.py +++ b/tests/parser/remote/test_crawler_loader.py @@ -13,8 +13,17 @@ class DummyResponse: return None +def _mock_validate_url(url): + """Mock validate_url that allows test URLs through.""" + from urllib.parse import urlparse + if not urlparse(url).scheme: + url = "http://" + url + return url + + +@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url) @patch("application.parser.remote.crawler_loader.requests.get") -def test_load_data_crawls_same_domain_links(mock_requests_get): +def test_load_data_crawls_same_domain_links(mock_requests_get, mock_validate_url): responses = { "http://example.com": DummyResponse( """ @@ -29,7 +38,7 @@ def test_load_data_crawls_same_domain_links(mock_requests_get): "http://example.com/about": DummyResponse("About page"), } - def response_side_effect(url: str): + def response_side_effect(url: str, timeout=30): if url not in responses: raise AssertionError(f"Unexpected request for URL: {url}") return responses[url] @@ -76,8 +85,9 @@ def test_load_data_crawls_same_domain_links(mock_requests_get): assert loader_call_order == ["http://example.com", "http://example.com/about"] +@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url) @patch("application.parser.remote.crawler_loader.requests.get") -def test_load_data_accepts_list_input_and_adds_scheme(mock_requests_get): +def test_load_data_accepts_list_input_and_adds_scheme(mock_requests_get, mock_validate_url): mock_requests_get.return_value = DummyResponse("No links here") doc = MagicMock(spec=LCDocument) @@ -92,7 +102,7 @@ def test_load_data_accepts_list_input_and_adds_scheme(mock_requests_get): result = crawler.load_data(["example.com", "unused.com"]) - mock_requests_get.assert_called_once_with("http://example.com") + mock_requests_get.assert_called_once_with("http://example.com", timeout=30) crawler.loader.assert_called_once_with(["http://example.com"]) assert len(result) == 1 @@ -100,8 +110,9 @@ def test_load_data_accepts_list_input_and_adds_scheme(mock_requests_get): assert result[0].extra_info == {"source": "http://example.com"} +@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url) @patch("application.parser.remote.crawler_loader.requests.get") -def test_load_data_respects_limit(mock_requests_get): +def test_load_data_respects_limit(mock_requests_get, mock_validate_url): responses = { "http://example.com": DummyResponse( """ @@ -115,7 +126,7 @@ def test_load_data_respects_limit(mock_requests_get): "http://example.com/about": DummyResponse("About"), } - mock_requests_get.side_effect = lambda url: responses[url] + mock_requests_get.side_effect = lambda url, timeout=30: responses[url] root_doc = MagicMock(spec=LCDocument) root_doc.page_content = "Root content" @@ -143,9 +154,10 @@ def test_load_data_respects_limit(mock_requests_get): assert crawler.loader.call_count == 1 +@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url) @patch("application.parser.remote.crawler_loader.logging") @patch("application.parser.remote.crawler_loader.requests.get") -def test_load_data_logs_and_skips_on_loader_error(mock_requests_get, mock_logging): +def test_load_data_logs_and_skips_on_loader_error(mock_requests_get, mock_logging, mock_validate_url): mock_requests_get.return_value = DummyResponse("Error route") failing_loader_instance = MagicMock() @@ -157,7 +169,7 @@ def test_load_data_logs_and_skips_on_loader_error(mock_requests_get, mock_loggin result = crawler.load_data("http://example.com") assert result == [] - mock_requests_get.assert_called_once_with("http://example.com") + mock_requests_get.assert_called_once_with("http://example.com", timeout=30) failing_loader_instance.load.assert_called_once() mock_logging.error.assert_called_once() @@ -165,3 +177,16 @@ def test_load_data_logs_and_skips_on_loader_error(mock_requests_get, mock_loggin assert "Error processing URL http://example.com" in message assert mock_logging.error.call_args.kwargs.get("exc_info") is True + +@patch("application.parser.remote.crawler_loader.validate_url") +def test_load_data_returns_empty_on_ssrf_validation_failure(mock_validate_url): + """Test that SSRF validation failure returns empty list.""" + from application.core.url_validation import SSRFError + mock_validate_url.side_effect = SSRFError("Access to private IP not allowed") + + crawler = CrawlerLoader() + result = crawler.load_data("http://192.168.1.1") + + assert result == [] + mock_validate_url.assert_called_once() + diff --git a/tests/parser/remote/test_crawler_markdown.py b/tests/parser/remote/test_crawler_markdown.py index ac27b3d0..b2b3f21c 100644 --- a/tests/parser/remote/test_crawler_markdown.py +++ b/tests/parser/remote/test_crawler_markdown.py @@ -1,5 +1,6 @@ from types import SimpleNamespace from unittest.mock import MagicMock +from urllib.parse import urlparse import pytest import requests @@ -29,6 +30,21 @@ def _fake_extract(value: str) -> SimpleNamespace: return SimpleNamespace(domain=domain, suffix=suffix) +def _mock_validate_url(url): + """Mock validate_url that allows test URLs through.""" + if not urlparse(url).scheme: + url = "http://" + url + return url + + +@pytest.fixture(autouse=True) +def _patch_validate_url(monkeypatch): + monkeypatch.setattr( + "application.parser.remote.crawler_markdown.validate_url", + _mock_validate_url, + ) + + @pytest.fixture(autouse=True) def _patch_tldextract(monkeypatch): monkeypatch.setattr( @@ -112,7 +128,7 @@ def test_load_data_allows_subdomains(_patch_markdownify): assert len(docs) == 2 -def test_load_data_handles_fetch_errors(monkeypatch, _patch_markdownify): +def test_load_data_handles_fetch_errors(monkeypatch, _patch_markdownify, _patch_validate_url): root_html = """ Home About @@ -137,3 +153,21 @@ def test_load_data_handles_fetch_errors(monkeypatch, _patch_markdownify): assert docs[0].text == "Home Markdown" assert mock_print.called + +def test_load_data_returns_empty_on_ssrf_validation_failure(monkeypatch): + """Test that SSRF validation failure returns empty list.""" + from application.core.url_validation import SSRFError + + def raise_ssrf_error(url): + raise SSRFError("Access to private IP not allowed") + + monkeypatch.setattr( + "application.parser.remote.crawler_markdown.validate_url", + raise_ssrf_error, + ) + + loader = CrawlerLoader() + result = loader.load_data("http://192.168.1.1") + + assert result == [] + diff --git a/tests/test_zip_extraction_security.py b/tests/test_zip_extraction_security.py new file mode 100644 index 00000000..c53452f7 --- /dev/null +++ b/tests/test_zip_extraction_security.py @@ -0,0 +1,293 @@ +"""Tests for zip extraction security measures.""" + +import os +import tempfile +import zipfile + +import pytest + +from application.worker import ( + ZipExtractionError, + _is_path_safe, + _validate_zip_safety, + extract_zip_recursive, + MAX_FILE_COUNT, +) + + +class TestIsPathSafe: + """Tests for _is_path_safe function.""" + + def test_safe_path_in_directory(self): + """Normal file within directory should be safe.""" + assert _is_path_safe("/tmp/extract", "/tmp/extract/file.txt") is True + + def test_safe_path_in_subdirectory(self): + """File in subdirectory should be safe.""" + assert _is_path_safe("/tmp/extract", "/tmp/extract/subdir/file.txt") is True + + def test_unsafe_path_parent_traversal(self): + """Path traversal to parent directory should be unsafe.""" + assert _is_path_safe("/tmp/extract", "/tmp/extract/../etc/passwd") is False + + def test_unsafe_path_absolute(self): + """Absolute path outside base should be unsafe.""" + assert _is_path_safe("/tmp/extract", "/etc/passwd") is False + + def test_unsafe_path_sibling(self): + """Sibling directory should be unsafe.""" + assert _is_path_safe("/tmp/extract", "/tmp/other/file.txt") is False + + def test_base_path_itself(self): + """Base path itself should be safe.""" + assert _is_path_safe("/tmp/extract", "/tmp/extract") is True + + +class TestValidateZipSafety: + """Tests for _validate_zip_safety function.""" + + def test_valid_small_zip(self): + """Small valid zip file should pass validation.""" + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "test.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a small valid zip + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("test.txt", "Hello, World!") + + # Should not raise + _validate_zip_safety(zip_path, extract_to) + + def test_zip_with_too_many_files(self): + """Zip with too many files should be rejected.""" + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "test.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a zip with many files (just over limit) + with zipfile.ZipFile(zip_path, "w") as zf: + for i in range(MAX_FILE_COUNT + 1): + zf.writestr(f"file_{i}.txt", "x") + + with pytest.raises(ZipExtractionError) as exc_info: + _validate_zip_safety(zip_path, extract_to) + assert "too many files" in str(exc_info.value).lower() + + def test_zip_with_path_traversal(self): + """Zip with path traversal attempt should be rejected.""" + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "test.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a zip with path traversal + with zipfile.ZipFile(zip_path, "w") as zf: + # Add a normal file first + zf.writestr("normal.txt", "normal content") + # Add a file with path traversal + zf.writestr("../../../etc/passwd", "malicious content") + + with pytest.raises(ZipExtractionError) as exc_info: + _validate_zip_safety(zip_path, extract_to) + assert "path traversal" in str(exc_info.value).lower() + + def test_corrupted_zip(self): + """Corrupted zip file should be rejected.""" + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "test.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a corrupted "zip" file + with open(zip_path, "wb") as f: + f.write(b"not a zip file content") + + with pytest.raises(ZipExtractionError) as exc_info: + _validate_zip_safety(zip_path, extract_to) + assert "invalid" in str(exc_info.value).lower() or "corrupted" in str(exc_info.value).lower() + + +class TestExtractZipRecursive: + """Tests for extract_zip_recursive function.""" + + def test_extract_valid_zip(self): + """Valid zip file should be extracted successfully.""" + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "test.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a valid zip + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("test.txt", "Hello, World!") + zf.writestr("subdir/nested.txt", "Nested content") + + extract_zip_recursive(zip_path, extract_to) + + # Check files were extracted + assert os.path.exists(os.path.join(extract_to, "test.txt")) + assert os.path.exists(os.path.join(extract_to, "subdir", "nested.txt")) + + # Check zip was removed + assert not os.path.exists(zip_path) + + def test_extract_nested_zip(self): + """Nested zip files should be extracted recursively.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create inner zip + inner_zip_content = b"" + with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as inner_tmp: + with zipfile.ZipFile(inner_tmp.name, "w") as inner_zf: + inner_zf.writestr("inner.txt", "Inner content") + with open(inner_tmp.name, "rb") as f: + inner_zip_content = f.read() + os.unlink(inner_tmp.name) + + # Create outer zip containing inner zip + zip_path = os.path.join(temp_dir, "outer.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("outer.txt", "Outer content") + zf.writestr("inner.zip", inner_zip_content) + + extract_zip_recursive(zip_path, extract_to) + + # Check outer file was extracted + assert os.path.exists(os.path.join(extract_to, "outer.txt")) + + # Check inner zip was extracted + assert os.path.exists(os.path.join(extract_to, "inner.txt")) + + # Check both zips were removed + assert not os.path.exists(zip_path) + assert not os.path.exists(os.path.join(extract_to, "inner.zip")) + + def test_respects_max_depth(self): + """Extraction should stop at max recursion depth.""" + with tempfile.TemporaryDirectory() as temp_dir: + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a chain of nested zips + current_content = b"Final content" + for i in range(7): # More than default max_depth of 5 + inner_tmp = tempfile.NamedTemporaryFile(suffix=".zip", delete=False) + with zipfile.ZipFile(inner_tmp.name, "w") as zf: + if i == 0: + zf.writestr("content.txt", current_content.decode()) + else: + zf.writestr("nested.zip", current_content) + with open(inner_tmp.name, "rb") as f: + current_content = f.read() + os.unlink(inner_tmp.name) + + # Write the final outermost zip + zip_path = os.path.join(temp_dir, "outer.zip") + with open(zip_path, "wb") as f: + f.write(current_content) + + # Extract with max_depth=2 + extract_zip_recursive(zip_path, extract_to, max_depth=2) + + # The deepest nested zips should remain unextracted + # (we can't easily verify the exact behavior, but the function should not crash) + + def test_rejects_path_traversal(self): + """Zip with path traversal should be rejected and removed.""" + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "malicious.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a malicious zip + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("../../../tmp/malicious.txt", "malicious") + + extract_zip_recursive(zip_path, extract_to) + + # Zip should be removed + assert not os.path.exists(zip_path) + + # Malicious file should NOT exist outside extract_to + assert not os.path.exists("/tmp/malicious.txt") + + def test_handles_corrupted_zip_gracefully(self): + """Corrupted zip should be handled gracefully without crashing.""" + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "corrupted.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a corrupted file + with open(zip_path, "wb") as f: + f.write(b"This is not a valid zip file") + + # Should not raise, just log error + extract_zip_recursive(zip_path, extract_to) + + # Function should complete without exception + + +class TestZipBombProtection: + """Tests specifically for zip bomb protection.""" + + def test_detects_high_compression_ratio(self): + """Highly compressed data should trigger compression ratio check.""" + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "bomb.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a file with highly compressible content (all zeros) + # This triggers the compression ratio check + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + # Create a large file with repetitive content - compresses extremely well + repetitive_content = "A" * (1024 * 1024) # 1 MB of 'A's + zf.writestr("repetitive.txt", repetitive_content) + + # This should be rejected due to high compression ratio + with pytest.raises(ZipExtractionError) as exc_info: + _validate_zip_safety(zip_path, extract_to) + assert "compression ratio" in str(exc_info.value).lower() + + def test_normal_compression_passes(self): + """Normal compression ratio should pass validation.""" + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "normal.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a zip with random-ish content that doesn't compress well + import random + random.seed(42) + random_content = "".join( + random.choices("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", k=10240) + ) + + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("random.txt", random_content) + + # Should pass - random content doesn't compress well + _validate_zip_safety(zip_path, extract_to) + + def test_size_limit_check(self): + """Files exceeding size limit should be rejected.""" + # Note: We can't easily create a real zip bomb in tests + # This test verifies the validation logic works + with tempfile.TemporaryDirectory() as temp_dir: + zip_path = os.path.join(temp_dir, "test.zip") + extract_to = os.path.join(temp_dir, "extract") + os.makedirs(extract_to) + + # Create a zip with a reasonable size (no compression to avoid ratio issues) + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_STORED) as zf: + # 10 KB file + zf.writestr("normal.txt", "x" * 10240) + + # Should pass + _validate_zip_safety(zip_path, extract_to) From 197e94302ba02c2ec6988030a6a228ce433fd3ab Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 24 Dec 2025 16:35:57 +0000 Subject: [PATCH 36/93] Patches (#2219) * feat: implement URL validation to prevent SSRF * feat: add zip extraction security * ruff fixes * fix: standardize error messages across API responses --- application/api/answer/routes/answer.py | 2 +- application/api/connector/routes.py | 120 ++++++++++++++++++------ application/api/user/sources/routes.py | 2 +- application/api/user/tools/mcp.py | 10 +- application/auth.py | 4 +- application/retriever/classic_rag.py | 4 +- frontend/.env.development | 4 +- 7 files changed, 102 insertions(+), 44 deletions(-) diff --git a/application/api/answer/routes/answer.py b/application/api/answer/routes/answer.py index e79ea378..c24ebffc 100644 --- a/application/api/answer/routes/answer.py +++ b/application/api/answer/routes/answer.py @@ -137,5 +137,5 @@ class AnswerResource(Resource, BaseAnswerResource): f"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}", extra={"error": str(e), "traceback": traceback.format_exc()}, ) - return make_response({"error": str(e)}, 500) + return make_response({"error": "An error occurred processing your request"}, 500) return make_response(result, 200) diff --git a/application/api/connector/routes.py b/application/api/connector/routes.py index d8efc1d3..1a4e80bf 100644 --- a/application/api/connector/routes.py +++ b/application/api/connector/routes.py @@ -1,7 +1,9 @@ import base64 import datetime +import html import json import uuid +from urllib.parse import urlencode from bson.objectid import ObjectId @@ -35,6 +37,18 @@ connector = Blueprint("connector", __name__) connectors_ns = Namespace("connectors", description="Connector operations", path="/") api.add_namespace(connectors_ns) +# Fixed callback status path to prevent open redirect +CALLBACK_STATUS_PATH = "/api/connectors/callback-status" + + +def build_callback_redirect(params: dict) -> str: + """Build a safe redirect URL to the callback status page. + + Uses a fixed path and properly URL-encodes all parameters + to prevent URL injection and open redirect vulnerabilities. + """ + return f"{CALLBACK_STATUS_PATH}?{urlencode(params)}" + @connectors_ns.route("/api/connectors/auth") @@ -75,8 +89,8 @@ class ConnectorAuth(Resource): "state": state }), 200) except Exception as e: - current_app.logger.error(f"Error generating connector auth URL: {e}") - return make_response(jsonify({"success": False, "error": str(e)}), 500) + current_app.logger.error(f"Error generating connector auth URL: {e}", exc_info=True) + return make_response(jsonify({"success": False, "error": "Failed to generate authorization URL"}), 500) @connectors_ns.route("/api/connectors/callback") @@ -93,18 +107,37 @@ class ConnectorsCallback(Resource): error = request.args.get('error') state_dict = json.loads(base64.urlsafe_b64decode(state.encode()).decode()) - provider = state_dict["provider"] - state_object_id = state_dict["object_id"] + provider = state_dict.get("provider") + state_object_id = state_dict.get("object_id") + + # Validate provider + if not provider or not isinstance(provider, str) or not ConnectorCreator.is_supported(provider): + return redirect(build_callback_redirect({ + "status": "error", + "message": "Invalid provider" + })) if error: if error == "access_denied": - return redirect(f"/api/connectors/callback-status?status=cancelled&message=Authentication+was+cancelled.+You+can+try+again+if+you'd+like+to+connect+your+account.&provider={provider}") + return redirect(build_callback_redirect({ + "status": "cancelled", + "message": "Authentication was cancelled. You can try again if you'd like to connect your account.", + "provider": provider + })) else: current_app.logger.warning(f"OAuth error in callback: {error}") - return redirect(f"/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.&provider={provider}") + return redirect(build_callback_redirect({ + "status": "error", + "message": "Authentication failed. Please try again and make sure to grant all requested permissions.", + "provider": provider + })) if not authorization_code: - return redirect(f"/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.&provider={provider}") + return redirect(build_callback_redirect({ + "status": "error", + "message": "Authentication failed. Please try again and make sure to grant all requested permissions.", + "provider": provider + })) try: auth = ConnectorCreator.create_auth(provider) @@ -141,15 +174,28 @@ class ConnectorsCallback(Resource): ) # Redirect to success page with session token and user email - return redirect(f"/api/connectors/callback-status?status=success&message=Authentication+successful&provider={provider}&session_token={session_token}&user_email={user_email}") + return redirect(build_callback_redirect({ + "status": "success", + "message": "Authentication successful", + "provider": provider, + "session_token": session_token, + "user_email": user_email + })) except Exception as e: current_app.logger.error(f"Error exchanging code for tokens: {str(e)}", exc_info=True) - return redirect(f"/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.&provider={provider}") + return redirect(build_callback_redirect({ + "status": "error", + "message": "Authentication failed. Please try again and make sure to grant all requested permissions.", + "provider": provider + })) except Exception as e: current_app.logger.error(f"Error handling connector callback: {e}") - return redirect("/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.") + return redirect(build_callback_redirect({ + "status": "error", + "message": "Authentication failed. Please try again and make sure to grant all requested permissions." + })) @connectors_ns.route("/api/connectors/files") @@ -228,8 +274,8 @@ class ConnectorFiles(Resource): "has_more": has_more }), 200) except Exception as e: - current_app.logger.error(f"Error loading connector files: {e}") - return make_response(jsonify({"success": False, "error": f"Failed to load files: {str(e)}"}), 500) + current_app.logger.error(f"Error loading connector files: {e}", exc_info=True) + return make_response(jsonify({"success": False, "error": "Failed to load files"}), 500) @connectors_ns.route("/api/connectors/validate-session") @@ -289,8 +335,8 @@ class ConnectorValidateSession(Resource): "access_token": token_info.get('access_token') }), 200) except Exception as e: - current_app.logger.error(f"Error validating connector session: {e}") - return make_response(jsonify({"success": False, "error": str(e)}), 500) + current_app.logger.error(f"Error validating connector session: {e}", exc_info=True) + return make_response(jsonify({"success": False, "error": "Failed to validate session"}), 500) @connectors_ns.route("/api/connectors/disconnect") @@ -311,8 +357,8 @@ class ConnectorDisconnect(Resource): return make_response(jsonify({"success": True}), 200) except Exception as e: - current_app.logger.error(f"Error disconnecting connector session: {e}") - return make_response(jsonify({"success": False, "error": str(e)}), 500) + current_app.logger.error(f"Error disconnecting connector session: {e}", exc_info=True) + return make_response(jsonify({"success": False, "error": "Failed to disconnect session"}), 500) @connectors_ns.route("/api/connectors/sync") @@ -418,8 +464,8 @@ class ConnectorSync(Resource): return make_response( jsonify({ "success": False, - "error": str(err) - }), + "error": "Failed to sync connector source" + }), 400 ) @@ -430,17 +476,28 @@ class ConnectorCallbackStatus(Resource): def get(self): """Return HTML page with connector authentication status""" try: - status = request.args.get('status', 'error') - message = request.args.get('message', '') - provider = request.args.get('provider', 'connector') + # Validate and sanitize status to a known value + status_raw = request.args.get('status', 'error') + status = status_raw if status_raw in ('success', 'error', 'cancelled') else 'error' + + # Escape all user-controlled values for HTML context + message = html.escape(request.args.get('message', '')) + provider_raw = request.args.get('provider', 'connector') + provider = html.escape(provider_raw.replace('_', ' ').title()) session_token = request.args.get('session_token', '') - user_email = request.args.get('user_email', '') - + user_email = html.escape(request.args.get('user_email', '')) + + # Use json.dumps for safe JavaScript string embedding + js_status = json.dumps(status) + js_session_token = json.dumps(session_token) + js_user_email = json.dumps(user_email) + js_provider_type = json.dumps(provider_raw) + html_content = f""" - {provider.replace('_', ' ').title()} Authentication + {provider} Authentication