From 8289b02ab0d533d45bf9dcf6a1af2dea4b003984 Mon Sep 17 00:00:00 2001 From: Siddhant Rai Date: Sat, 26 Apr 2025 12:00:29 +0530 Subject: [PATCH 1/5] feat: add agent webhook endpoint and implement related functionality --- application/api/user/routes.py | 90 +++++++- application/api/user/tasks.py | 14 +- application/worker.py | 247 +++++++++++++++++----- frontend/src/Navigation.tsx | 47 ++-- frontend/src/agents/AgentPreview.tsx | 1 + frontend/src/agents/NewAgent.tsx | 11 +- frontend/src/agents/index.tsx | 29 ++- frontend/src/api/endpoints.ts | 1 + frontend/src/api/services/userService.ts | 2 + frontend/src/components/MessageInput.tsx | 13 +- frontend/src/modals/AgentDetailsModal.tsx | 59 +++++- frontend/src/modals/ConfirmationModal.tsx | 10 +- 12 files changed, 424 insertions(+), 100 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 8876be6b..391444fc 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -4,6 +4,7 @@ import math import os import shutil import uuid +import secrets from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef @@ -14,7 +15,12 @@ from werkzeug.utils import secure_filename from application.agents.tools.tool_manager import ToolManager -from application.api.user.tasks import ingest, ingest_remote, store_attachment +from application.api.user.tasks import ( + ingest, + ingest_remote, + store_attachment, + process_agent_webhook, +) from application.core.mongo_db import MongoDB from application.core.settings import settings from application.extensions import api @@ -1329,6 +1335,88 @@ class DeleteAgent(Resource): return make_response(jsonify({"id": deleted_id}), 200) +@user_ns.route("/api/agent_webhook") +class AgentWebhook(Resource): + @api.doc( + params={"id": "ID of the agent"}, + description="Generate webhook URL for the agent", + ) + def get(self): + decoded_token = request.decoded_token + if not decoded_token: + return make_response(jsonify({"success": False}), 401) + user = decoded_token.get("sub") + agent_id = request.args.get("id") + if not agent_id: + return make_response( + jsonify({"success": False, "message": "ID is required"}), 400 + ) + + try: + agent = agents_collection.find_one( + {"_id": ObjectId(agent_id), "user": user} + ) + if not agent: + return make_response( + jsonify({"success": False, "message": "Agent not found"}), 404 + ) + + webhook_token = agent.get("incoming_webhook_token") + if not webhook_token: + webhook_token = secrets.token_urlsafe(32) + agents_collection.update_one( + {"_id": ObjectId(agent_id), "user": user}, + {"$set": {"incoming_webhook_token": webhook_token}}, + ) + base_url = settings.API_URL.rstrip("/") + full_webhook_url = f"{base_url}/api/webhooks/agents/{webhook_token}" + + except Exception as err: + current_app.logger.error(f"Error generating webhook URL: {err}") + return make_response( + jsonify({"success": False, "message": "Error generating webhook URL"}), + 400, + ) + return make_response( + jsonify({"success": True, "webhook_url": full_webhook_url}), 200 + ) + + +@user_ns.route(f"/api/webhooks/agents/") +class AgentWebhookListener(Resource): + @api.doc(description="Webhook listener for agent events") + def post(self, webhook_token): + agent = agents_collection.find_one( + {"incoming_webhook_token": webhook_token}, {"_id": 1} + ) + if not agent: + return make_response( + jsonify({"success": False, "message": "Agent not found"}), 404 + ) + data = request.get_json() + if not data: + return make_response( + jsonify({"success": False, "message": "No data provided"}), 400 + ) + + agent_id_str = str(agent["_id"]) + current_app.logger.info( + f"Incoming webhook received for agent {agent_id_str}. Enqueuing task." + ) + + try: + task = process_agent_webhook.delay( + agent_id=agent_id_str, + payload=data, + ) + except Exception as err: + current_app.logger.error(f"Error processing webhook: {err}") + return make_response( + jsonify({"success": False, "message": "Error processing webhook"}), 400 + ) + return make_response(jsonify({"success": True, "task_id": task.id}), 200) + + @user_ns.route("/api/share") class ShareConversation(Resource): share_conversation_model = api.model( diff --git a/application/api/user/tasks.py b/application/api/user/tasks.py index 24cff3c6..f53d856b 100644 --- a/application/api/user/tasks.py +++ b/application/api/user/tasks.py @@ -1,7 +1,13 @@ from datetime import timedelta from application.celery_init import celery -from application.worker import ingest_worker, remote_worker, sync_worker, attachment_worker +from application.worker import ( + agent_webhook_worker, + attachment_worker, + ingest_worker, + remote_worker, + sync_worker, +) @celery.task(bind=True) @@ -28,6 +34,12 @@ def store_attachment(self, directory, saved_files, user): return resp +@celery.task(bind=True) +def process_agent_webhook(self, agent_id, payload): + resp = agent_webhook_worker(self, agent_id, payload) + return resp + + @celery.on_after_configure.connect def setup_periodic_tasks(sender, **kwargs): sender.add_periodic_task( diff --git a/application/worker.py b/application/worker.py index bbd422ac..4782a83b 100755 --- a/application/worker.py +++ b/application/worker.py @@ -1,3 +1,4 @@ +import json import logging import os import shutil @@ -7,15 +8,20 @@ from collections import Counter from urllib.parse import urljoin import requests +from bson.dbref import DBRef from bson.objectid import ObjectId +from application.agents.agent_creator import AgentCreator +from application.api.answer.routes import get_prompt + from application.core.mongo_db import MongoDB from application.core.settings import settings -from application.parser.file.bulk import SimpleDirectoryReader +from application.parser.chunking import Chunker from application.parser.embedding_pipeline import embed_and_store_documents +from application.parser.file.bulk import SimpleDirectoryReader from application.parser.remote.remote_creator import RemoteCreator from application.parser.schema.base import Document -from application.parser.chunking import Chunker +from application.retriever.retriever_creator import RetrieverCreator from application.utils import count_tokens_docs mongo = MongoDB.get_client() @@ -27,18 +33,22 @@ MIN_TOKENS = 150 MAX_TOKENS = 1250 RECURSION_DEPTH = 2 + # Define a function to extract metadata from a given filename. def metadata_from_filename(title): return {"title": title} + # Define a function to generate a random string of a given length. def generate_random_string(length): return "".join([string.ascii_letters[i % 52] for i in range(length)]) + current_dir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ) + def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5): """ Recursively extract zip files with a limit on recursion depth. @@ -69,6 +79,7 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5): file_path = os.path.join(root, file) extract_zip_recursive(file_path, root, current_depth + 1, max_depth) + def download_file(url, params, dest_path): try: response = requests.get(url, params=params) @@ -79,6 +90,7 @@ def download_file(url, params, dest_path): logging.error(f"Error downloading file: {e}") raise + def upload_index(full_path, file_data): try: if settings.VECTOR_STORE == "faiss": @@ -87,7 +99,9 @@ def upload_index(full_path, file_data): "file_pkl": open(full_path + "/index.pkl", "rb"), } response = requests.post( - urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data + urljoin(settings.API_URL, "/api/upload_index"), + files=files, + data=file_data, ) else: response = requests.post( @@ -102,6 +116,75 @@ def upload_index(full_path, file_data): for file in files.values(): file.close() + +def run_agent_logic(agent_config, input_data): + try: + source = agent_config.get("source") + retriever = agent_config.get("retriever", "classic") + if isinstance(source, DBRef): + source_doc = db.dereference(source) + source = str(source_doc["_id"]) + retriever = source_doc.get("retriever", agent_config.get("retriever")) + else: + source = {} + source = {"active_docs": source} + chunks = int(agent_config.get("chunks", 2)) + prompt_id = agent_config.get("prompt_id", "default") + user_api_key = agent_config["key"] + agent_type = agent_config.get("agent_type", "classic") + decoded_token = {"sub": agent_config.get("user")} + prompt = get_prompt(prompt_id) + agent = AgentCreator.create_agent( + agent_type, + endpoint="webhook", + llm_name=settings.LLM_NAME, + gpt_model=settings.MODEL_NAME, + api_key=settings.API_KEY, + user_api_key=user_api_key, + prompt=prompt, + chat_history=[], + decoded_token=decoded_token, + attachments=[], + ) + retriever = RetrieverCreator.create_retriever( + retriever, + source=source, + chat_history=[], + prompt=prompt, + chunks=chunks, + token_limit=settings.DEFAULT_MAX_HISTORY, + gpt_model=settings.MODEL_NAME, + user_api_key=user_api_key, + decoded_token=decoded_token, + ) + answer = agent.gen(query=input_data, retriever=retriever) + response_full = "" + thought = "" + source_log_docs = [] + tool_calls = [] + + for line in answer: + if "answer" in line: + response_full += str(line["answer"]) + elif "sources" in line: + source_log_docs.extend(line["sources"]) + elif "tool_calls" in line: + tool_calls.extend(line["tool_calls"]) + elif "thought" in line: + thought += line["thought"] + + result = { + "answer": response_full, + "sources": source_log_docs, + "tool_calls": tool_calls, + "thought": thought, + } + return result + except Exception as e: + logging.error(f"Error in run_agent_logic: {e}", exc_info=True) + raise + + # Define the main function for ingesting and processing documents. def ingest_worker( self, directory, formats, name_job, filename, user, retriever="classic" @@ -133,7 +216,11 @@ def ingest_worker( if not os.path.exists(full_path): os.makedirs(full_path) - download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename)) + download_file( + urljoin(settings.API_URL, "/api/download"), + file_data, + os.path.join(full_path, filename), + ) # check if file is .zip and extract it if filename.endswith(".zip"): @@ -157,7 +244,7 @@ def ingest_worker( chunking_strategy="classic_chunk", max_tokens=MAX_TOKENS, min_tokens=MIN_TOKENS, - duplicate_headers=False + duplicate_headers=False, ) raw_docs = chunker.chunk(documents=raw_docs) @@ -172,12 +259,14 @@ def ingest_worker( for i in range(min(5, len(raw_docs))): logging.info(f"Sample document {i}: {raw_docs[i]}") - file_data.update({ - "tokens": tokens, - "retriever": retriever, - "id": str(id), - "type": "local", - }) + file_data.update( + { + "tokens": tokens, + "retriever": retriever, + "id": str(id), + "type": "local", + } + ) upload_index(full_path, file_data) # delete local @@ -192,6 +281,7 @@ def ingest_worker( "limited": False, } + def remote_worker( self, source_data, @@ -203,7 +293,7 @@ def remote_worker( sync_frequency="never", operation_mode="upload", doc_id=None, -): +): full_path = os.path.join(directory, user, name_job) if not os.path.exists(full_path): os.makedirs(full_path) @@ -218,7 +308,7 @@ def remote_worker( chunking_strategy="classic_chunk", max_tokens=MAX_TOKENS, min_tokens=MIN_TOKENS, - duplicate_headers=False + duplicate_headers=False, ) docs = chunker.chunk(documents=raw_docs) docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs] @@ -260,6 +350,7 @@ def remote_worker( logging.info("remote_worker task completed successfully") return {"urls": source_data, "name_job": name_job, "user": user, "limited": False} + def sync( self, source_data, @@ -289,6 +380,7 @@ def sync( return {"status": "error", "error": str(e)} return {"status": "success"} + def sync_worker(self, frequency): sync_counts = Counter() sources = sources_collection.find() @@ -313,84 +405,137 @@ def sync_worker(self, frequency): for key in ["total_sync_count", "sync_success", "sync_failure"] } + def attachment_worker(self, directory, file_info, user): """ Process and store a single attachment without vectorization. - + Args: self: Reference to the instance of the task. directory (str): Base directory for storing files. file_info (dict): Dictionary with folder and filename info. user (str): User identifier. - + Returns: dict: Information about processed attachment. """ import datetime - import os import mimetypes + import os + from application.utils import num_tokens_from_string - + mongo = MongoDB.get_client() db = mongo["docsgpt"] attachments_collection = db["attachments"] - + filename = file_info["filename"] attachment_id = file_info["attachment_id"] - - logging.info(f"Processing attachment: {attachment_id}/{filename}", extra={"user": user}) - + + logging.info( + f"Processing attachment: {attachment_id}/{filename}", extra={"user": user} + ) + self.update_state(state="PROGRESS", meta={"current": 10}) - + file_path = os.path.join(directory, filename) - + if not os.path.exists(file_path): logging.warning(f"File not found: {file_path}", extra={"user": user}) raise FileNotFoundError(f"File not found: {file_path}") - + try: - reader = SimpleDirectoryReader( - input_files=[file_path] - ) + reader = SimpleDirectoryReader(input_files=[file_path]) documents = reader.load_data() - + self.update_state(state="PROGRESS", meta={"current": 50}) - + if documents: content = documents[0].text token_count = num_tokens_from_string(content) - + file_path_relative = f"{settings.UPLOAD_FOLDER}/{user}/attachments/{attachment_id}/{filename}" - - mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' - + + mime_type = mimetypes.guess_type(file_path)[0] or "application/octet-stream" + doc_id = ObjectId(attachment_id) - attachments_collection.insert_one({ - "_id": doc_id, - "user": user, - "path": file_path_relative, - "content": content, - "token_count": token_count, - "mime_type": mime_type, - "date": datetime.datetime.now(), - }) - - logging.info(f"Stored attachment with ID: {attachment_id}", - extra={"user": user}) - + attachments_collection.insert_one( + { + "_id": doc_id, + "user": user, + "path": file_path_relative, + "content": content, + "token_count": token_count, + "mime_type": mime_type, + "date": datetime.datetime.now(), + } + ) + + logging.info( + f"Stored attachment with ID: {attachment_id}", extra={"user": user} + ) + self.update_state(state="PROGRESS", meta={"current": 100}) - + return { "filename": filename, "path": file_path_relative, "token_count": token_count, "attachment_id": attachment_id, - "mime_type": mime_type + "mime_type": mime_type, } else: - logging.warning("No content was extracted from the file", - extra={"user": user}) + logging.warning( + "No content was extracted from the file", extra={"user": user} + ) raise ValueError("No content was extracted from the file") except Exception as e: - logging.error(f"Error processing file {filename}: {e}", extra={"user": user}, exc_info=True) + logging.error( + f"Error processing file {filename}: {e}", + extra={"user": user}, + exc_info=True, + ) raise + + +def agent_webhook_worker(self, agent_id, payload): + """ + Process the webhook payload for an agent. + + Args: + self: Reference to the instance of the task. + agent_id (str): Unique identifier for the agent. + payload (dict): The payload data from the webhook. + + Returns: + dict: Information about the processed webhook. + """ + mongo = MongoDB.get_client() + db = mongo["docsgpt"] + agents_collection = db["agents"] + + self.update_state(state="PROGRESS", meta={"current": 1}) + try: + agent_oid = ObjectId(agent_id) + agent_config = agents_collection.find_one({"_id": agent_oid}) + if not agent_config: + raise ValueError(f"Agent with ID {agent_id} not found.") + input_data = payload.get("query", "") + if input_data is None or not isinstance(input_data, str): + input_data = json.dumps(payload) + except Exception as e: + logging.error(f"Error processing agent webhook: {e}", exc_info=True) + return {"status": "error", "error": str(e)} + + self.update_state(state="PROGRESS", meta={"current": 50}) + try: + result = run_agent_logic(agent_config, input_data) + except Exception as e: + logging.error(f"Error running agent logic: {e}", exc_info=True) + return {"status": "error", "error": str(e)} + finally: + self.update_state(state="PROGRESS", meta={"current": 100}) + logging.info( + f"Webhook processed for agent {agent_id}", extra={"agent_id": agent_id} + ) + return {"status": "success", "result": result} diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index 0e357a6d..53487dd6 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -44,6 +44,7 @@ import { setModalStateDeleteConv, setSelectedAgent, setAgents, + selectAgents, } from './preferences/preferenceSlice'; import Upload from './upload/Upload'; @@ -63,6 +64,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { const conversations = useSelector(selectConversations); const conversationId = useSelector(selectConversationId); const modalStateDeleteConv = useSelector(selectModalStateDeleteConv); + const agents = useSelector(selectAgents); const selectedAgent = useSelector(selectSelectedAgent); const { isMobile } = useMediaQuery(); @@ -76,6 +78,31 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { const navRef = useRef(null); + async function fetchRecentAgents() { + try { + let recentAgents: Agent[] = []; + if (!agents) { + const response = await userService.getAgents(token); + if (!response.ok) throw new Error('Failed to fetch agents'); + const data: Agent[] = await response.json(); + dispatch(setAgents(data)); + recentAgents = data; + } else recentAgents = agents; + setRecentAgents( + recentAgents + .filter((agent: Agent) => agent.status === 'published') + .sort( + (a: Agent, b: Agent) => + new Date(b.last_used_at ?? 0).getTime() - + new Date(a.last_used_at ?? 0).getTime(), + ) + .slice(0, 3), + ); + } catch (error) { + console.error('Failed to fetch recent agents: ', error); + } + } + async function fetchConversations() { dispatch(setConversations({ ...conversations, loading: true })); return await getConversations(token) @@ -88,25 +115,11 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { }); } - async function getAgents() { - const response = await userService.getAgents(token); - if (!response.ok) throw new Error('Failed to fetch agents'); - const data: Agent[] = await response.json(); - dispatch(setAgents(data)); - setRecentAgents( - data - .filter((agent: Agent) => agent.status === 'published') - .sort( - (a: Agent, b: Agent) => - new Date(b.last_used_at ?? 0).getTime() - - new Date(a.last_used_at ?? 0).getTime(), - ) - .slice(0, 3), - ); - } + useEffect(() => { + if (token) fetchRecentAgents(); + }, [agents, token, dispatch]); useEffect(() => { - if (recentAgents.length === 0) getAgents(); if (!conversations?.data) fetchConversations(); if (queries.length === 0) resetConversation(); }, [conversations?.data, dispatch]); diff --git a/frontend/src/agents/AgentPreview.tsx b/frontend/src/agents/AgentPreview.tsx index 5eaf10a9..621ac477 100644 --- a/frontend/src/agents/AgentPreview.tsx +++ b/frontend/src/agents/AgentPreview.tsx @@ -141,6 +141,7 @@ export default function AgentPreview() { loading={status === 'loading'} showSourceButton={selectedAgent ? false : true} showToolButton={selectedAgent ? false : true} + autoFocus={false} />

This is a preview of the agent. You can publish it to start using it diff --git a/frontend/src/agents/NewAgent.tsx b/frontend/src/agents/NewAgent.tsx index 37466a86..3aa1bf7d 100644 --- a/frontend/src/agents/NewAgent.tsx +++ b/frontend/src/agents/NewAgent.tsx @@ -155,9 +155,10 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { const data = await response.json(); if (data.id) setAgent((prev) => ({ ...prev, id: data.id })); if (data.key) setAgent((prev) => ({ ...prev, key: data.key })); - if (effectiveMode === 'new') { - setAgentDetails('ACTIVE'); + if (effectiveMode === 'new' || effectiveMode === 'draft') { setEffectiveMode('edit'); + setAgent((prev) => ({ ...prev, status: 'published' })); + setAgentDetails('ACTIVE'); } }; @@ -408,7 +409,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { agent.prompt_id ? prompts.filter( (prompt) => prompt.id === agent.prompt_id, - )[0].name || null + )[0]?.name || null : null } onSelect={(option: { label: string; value: string }) => @@ -532,7 +533,7 @@ function AgentPreviewArea() { const selectedAgent = useSelector(selectSelectedAgent); return (

- {selectedAgent?.id ? ( + {selectedAgent?.status === 'published' ? (
@@ -540,7 +541,7 @@ function AgentPreviewArea() {
{' '}

- Published agents can be previewd here + Published agents can be previewed here

)} diff --git a/frontend/src/agents/index.tsx b/frontend/src/agents/index.tsx index 49123cd6..0ceef669 100644 --- a/frontend/src/agents/index.tsx +++ b/frontend/src/agents/index.tsx @@ -12,7 +12,13 @@ import ThreeDots from '../assets/three-dots.svg'; import ContextMenu, { MenuOption } from '../components/ContextMenu'; import ConfirmationModal from '../modals/ConfirmationModal'; import { ActiveState } from '../models/misc'; -import { selectToken, setSelectedAgent } from '../preferences/preferenceSlice'; +import { + selectToken, + setSelectedAgent, + setAgents, + selectAgents, + selectSelectedAgent, +} from '../preferences/preferenceSlice'; import AgentLogs from './AgentLogs'; import NewAgent from './NewAgent'; import { Agent } from './types'; @@ -31,9 +37,12 @@ export default function Agents() { function AgentsList() { const navigate = useNavigate(); + const dispatch = useDispatch(); const token = useSelector(selectToken); + const agents = useSelector(selectAgents); + const selectedAgent = useSelector(selectSelectedAgent); - const [userAgents, setUserAgents] = useState([]); + const [userAgents, setUserAgents] = useState(agents || []); const [loading, setLoading] = useState(true); const getAgents = async () => { @@ -43,6 +52,7 @@ function AgentsList() { if (!response.ok) throw new Error('Failed to fetch agents'); const data = await response.json(); setUserAgents(data); + dispatch(setAgents(data)); setLoading(false); } catch (error) { console.error('Error:', error); @@ -52,6 +62,7 @@ function AgentsList() { useEffect(() => { getAgents(); + if (selectedAgent) dispatch(setSelectedAgent(null)); }, [token]); return (
@@ -62,6 +73,7 @@ function AgentsList() { Discover and create custom versions of DocsGPT that combine instructions, extra knowledge, and any combination of skills.

+ {/* Premade agents section */} {/*

Premade by DocsGPT @@ -200,8 +212,10 @@ function AgentCard({ ]; const handleClick = () => { - dispatch(setSelectedAgent(agent)); - navigate(`/`); + if (agent.status === 'published') { + dispatch(setSelectedAgent(agent)); + navigate(`/`); + } }; const handleDelete = async (agentId: string) => { @@ -214,8 +228,11 @@ function AgentCard({ }; return (
handleClick()} + className={`relative flex h-44 w-48 flex-col justify-between rounded-[1.2rem] bg-[#F6F6F6] px-6 py-5 hover:bg-[#ECECEC] dark:bg-[#383838] hover:dark:bg-[#383838]/80 ${agent.status === 'published' && 'cursor-pointer'}`} + onClick={(e) => { + e.stopPropagation(); + handleClick(); + }} >
`/api/update_agent/${agent_id}`, DELETE_AGENT: (id: string) => `/api/delete_agent?id=${id}`, + AGENT_WEBHOOK: (id: string) => `/api/agent_webhook?id=${id}`, PROMPTS: '/api/get_prompts', CREATE_PROMPT: '/api/create_prompt', DELETE_PROMPT: '/api/delete_prompt', diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index bbe20b10..4a0f45d8 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -31,6 +31,8 @@ const userService = { apiClient.put(endpoints.USER.UPDATE_AGENT(agent_id), data, token), deleteAgent: (id: string, token: string | null): Promise => apiClient.delete(endpoints.USER.DELETE_AGENT(id), token), + getAgentWebhook: (id: string, token: string | null): Promise => + apiClient.get(endpoints.USER.AGENT_WEBHOOK(id), token), getPrompts: (token: string | null): Promise => apiClient.get(endpoints.USER.PROMPTS, token), createPrompt: (data: any, token: string | null): Promise => diff --git a/frontend/src/components/MessageInput.tsx b/frontend/src/components/MessageInput.tsx index e7ef7f9d..60cd4b81 100644 --- a/frontend/src/components/MessageInput.tsx +++ b/frontend/src/components/MessageInput.tsx @@ -36,15 +36,7 @@ type MessageInputProps = { loading: boolean; showSourceButton?: boolean; showToolButton?: boolean; -}; - -type UploadState = { - taskId: string; - fileName: string; - progress: number; - attachment_id?: string; - token_count?: number; - status: 'uploading' | 'processing' | 'completed' | 'failed'; + autoFocus?: boolean; }; export default function MessageInput({ @@ -54,6 +46,7 @@ export default function MessageInput({ loading, showSourceButton = true, showToolButton = true, + autoFocus = true, }: MessageInputProps) { const { t } = useTranslation(); const [isDarkTheme] = useDarkTheme(); @@ -235,7 +228,7 @@ export default function MessageInput({ }; useEffect(() => { - inputRef.current?.focus(); + if (autoFocus) inputRef.current?.focus(); handleInput(); }, []); diff --git a/frontend/src/modals/AgentDetailsModal.tsx b/frontend/src/modals/AgentDetailsModal.tsx index 377dd7bd..c1a8c131 100644 --- a/frontend/src/modals/AgentDetailsModal.tsx +++ b/frontend/src/modals/AgentDetailsModal.tsx @@ -1,7 +1,12 @@ +import { useState } from 'react'; +import { useSelector } from 'react-redux'; + import { Agent } from '../agents/types'; import { ActiveState } from '../models/misc'; import WrapperModal from './WrapperModal'; -import { useNavigate } from 'react-router-dom'; +import userService from '../api/services/userService'; +import { selectToken } from '../preferences/preferenceSlice'; +import Spinner from '../components/Spinner'; type AgentDetailsModalProps = { agent: Agent; @@ -16,13 +21,41 @@ export default function AgentDetailsModal({ modalState, setModalState, }: AgentDetailsModalProps) { - const navigate = useNavigate(); + const token = useSelector(selectToken); + + const [publicLink, setPublicLink] = useState(null); + const [apiKey, setApiKey] = useState(null); + const [webhookUrl, setWebhookUrl] = useState(null); + const [loadingStates, setLoadingStates] = useState({ + publicLink: false, + apiKey: false, + webhook: false, + }); + + const setLoading = ( + key: 'publicLink' | 'apiKey' | 'webhook', + state: boolean, + ) => { + setLoadingStates((prev) => ({ ...prev, [key]: state })); + }; + + const handleGenerateWebhook = async () => { + setLoading('webhook', true); + const response = await userService.getAgentWebhook(agent.id ?? '', token); + if (!response.ok) { + setLoading('webhook', false); + return; + } + const data = await response.json(); + setWebhookUrl(data.webhook_url); + setLoading('webhook', false); + }; + if (modalState !== 'ACTIVE') return null; return ( { - // if (mode === 'new') navigate('/agents'); setModalState('INACTIVE'); }} > @@ -57,9 +90,23 @@ export default function AgentDetailsModal({

Webhooks

- + {webhookUrl ? ( +
+ + {webhookUrl} + + +
+ ) : ( + + )}

diff --git a/frontend/src/modals/ConfirmationModal.tsx b/frontend/src/modals/ConfirmationModal.tsx index 25f8c2da..28151736 100644 --- a/frontend/src/modals/ConfirmationModal.tsx +++ b/frontend/src/modals/ConfirmationModal.tsx @@ -40,19 +40,23 @@ export default function ConfirmationModal({ >
-

+

{message}

)} diff --git a/frontend/src/agents/index.tsx b/frontend/src/agents/index.tsx index 0ceef669..c2edb34a 100644 --- a/frontend/src/agents/index.tsx +++ b/frontend/src/agents/index.tsx @@ -138,6 +138,7 @@ function AgentsList() { )) @@ -160,9 +161,11 @@ function AgentsList() { function AgentCard({ agent, + agents, setUserAgents, }: { agent: Agent; + agents: Agent[]; setUserAgents: React.Dispatch>; }) { const navigate = useNavigate(); @@ -225,6 +228,7 @@ function AgentCard({ setUserAgents((prevAgents) => prevAgents.filter((prevAgent) => prevAgent.id !== data.id), ); + dispatch(setAgents(agents.filter((prevAgent) => prevAgent.id !== data.id))); }; return (
+ + diff --git a/frontend/src/assets/monitoring-white.svg b/frontend/src/assets/monitoring-white.svg new file mode 100644 index 00000000..b015eeee --- /dev/null +++ b/frontend/src/assets/monitoring-white.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/components/CopyButton.tsx b/frontend/src/components/CopyButton.tsx index c430603f..0afbbe82 100644 --- a/frontend/src/components/CopyButton.tsx +++ b/frontend/src/components/CopyButton.tsx @@ -1,58 +1,136 @@ +import clsx from 'clsx'; import copy from 'copy-to-clipboard'; -import { useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import CheckMark from '../assets/checkmark.svg?react'; -import Copy from '../assets/copy.svg?react'; +import CopyIcon from '../assets/copy.svg?react'; + +type CopyButtonProps = { + textToCopy: string; + bgColorLight?: string; + bgColorDark?: string; + hoverBgColorLight?: string; + hoverBgColorDark?: string; + iconSize?: string; + padding?: string; + showText?: boolean; + copiedDuration?: number; + className?: string; + iconWrapperClassName?: string; + textClassName?: string; +}; + +const DEFAULT_ICON_SIZE = 'w-4 h-4'; +const DEFAULT_PADDING = 'p-2'; +const DEFAULT_COPIED_DURATION = 2000; +const DEFAULT_BG_LIGHT = '#FFFFFF'; +const DEFAULT_BG_DARK = 'transparent'; +const DEFAULT_HOVER_BG_LIGHT = '#EEEEEE'; +const DEFAULT_HOVER_BG_DARK = '#4A4A4A'; export default function CopyButton({ - text, - colorLight, - colorDark, + textToCopy, + bgColorLight = DEFAULT_BG_LIGHT, + bgColorDark = DEFAULT_BG_DARK, + hoverBgColorLight = DEFAULT_HOVER_BG_LIGHT, + hoverBgColorDark = DEFAULT_HOVER_BG_DARK, + iconSize = DEFAULT_ICON_SIZE, + padding = DEFAULT_PADDING, showText = false, -}: { - text: string; - colorLight?: string; - colorDark?: string; - showText?: boolean; -}) { + copiedDuration = DEFAULT_COPIED_DURATION, + className, + iconWrapperClassName, + textClassName, +}: CopyButtonProps) { const { t } = useTranslation(); - const [copied, setCopied] = useState(false); - const [isCopyHovered, setIsCopyHovered] = useState(false); + const [isCopied, setIsCopied] = useState(false); + const timeoutIdRef = useRef(null); - const handleCopyClick = (text: string) => { - copy(text); - setCopied(true); - setTimeout(() => { - setCopied(false); - }, 3000); - }; + const iconWrapperClasses = clsx( + 'flex items-center justify-center rounded-full transition-colors duration-150 ease-in-out', + padding, + `bg-[${bgColorLight}] dark:bg-[${bgColorDark}]`, + `hover:bg-[${hoverBgColorLight}] dark:hover:bg-[${hoverBgColorDark}]`, + { + 'bg-green-100 dark:bg-green-900 hover:bg-green-100 dark:hover:bg-green-900': + isCopied, + }, + iconWrapperClassName, + ); + const rootButtonClasses = clsx( + 'flex items-center gap-2 group', + 'focus:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-blue-500 rounded-full', + className, + ); + + const textSpanClasses = clsx( + 'text-xs text-gray-600 dark:text-gray-400 transition-opacity duration-150 ease-in-out', + { 'opacity-75': isCopied }, + textClassName, + ); + + const IconComponent = isCopied ? CheckMark : CopyIcon; + const iconClasses = clsx(iconSize, { + 'stroke-green-600 dark:stroke-green-400': isCopied, + 'fill-none text-gray-700 dark:text-gray-300': !isCopied, + }); + + const buttonTitle = isCopied + ? t('conversation.copied') + : t('conversation.copy'); + const displayedText = isCopied + ? t('conversation.copied') + : t('conversation.copy'); + + const handleCopy = useCallback(() => { + if (isCopied) return; + + try { + const success = copy(textToCopy); + if (success) { + setIsCopied(true); + + if (timeoutIdRef.current) { + clearTimeout(timeoutIdRef.current); + } + + timeoutIdRef.current = setTimeout(() => { + setIsCopied(false); + timeoutIdRef.current = null; + }, copiedDuration); + } else { + console.warn('Copy command failed.'); + } + } catch (error) { + console.error('Failed to copy text:', error); + } + }, [textToCopy, copiedDuration, isCopied]); + + useEffect(() => { + return () => { + if (timeoutIdRef.current) { + clearTimeout(timeoutIdRef.current); + } + }; + }, []); return ( ); } diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx index a241b2d3..a7c8467d 100644 --- a/frontend/src/conversation/ConversationBubble.tsx +++ b/frontend/src/conversation/ConversationBubble.tsx @@ -5,10 +5,7 @@ import { useTranslation } from 'react-i18next'; import ReactMarkdown from 'react-markdown'; import { useSelector } from 'react-redux'; import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'; -import { - oneLight, - vscDarkPlus, -} from 'react-syntax-highlighter/dist/cjs/styles/prism'; +import { oneLight, vscDarkPlus } from 'react-syntax-highlighter/dist/cjs/styles/prism'; import rehypeKatex from 'rehype-katex'; import remarkGfm from 'remark-gfm'; import remarkMath from 'remark-math'; @@ -29,10 +26,7 @@ import CopyButton from '../components/CopyButton'; import Sidebar from '../components/Sidebar'; import SpeakButton from '../components/TextToSpeechButton'; import { useDarkTheme, useOutsideAlerter } from '../hooks'; -import { - selectChunks, - selectSelectedDocs, -} from '../preferences/preferenceSlice'; +import { selectChunks, selectSelectedDocs } from '../preferences/preferenceSlice'; import classes from './ConversationBubble.module.css'; import { FEEDBACK, MESSAGE_TYPE } from './conversationModels'; import { ToolCallsType } from './types'; @@ -377,7 +371,7 @@ const ConversationBubble = forwardRef< {language}
- +
{' '}

@@ -689,7 +683,7 @@ function ToolCalls({ toolCalls }: { toolCalls: ToolCallsType[] }) { Response {' '}

@@ -766,7 +760,7 @@ function Thought({ {language}

-

- Webhooks -

+
+

+ Webhook URL +

+ {webhookUrl && ( +
+ +
+ )} +
{webhookUrl ? ( -
- +
+

{webhookUrl} - - +

) : ( )}
diff --git a/frontend/src/settings/Analytics.tsx b/frontend/src/settings/Analytics.tsx index 04bec5c2..535200ef 100644 --- a/frontend/src/settings/Analytics.tsx +++ b/frontend/src/settings/Analytics.tsx @@ -1,11 +1,5 @@ import { - BarElement, - CategoryScale, - Chart as ChartJS, - Legend, - LinearScale, - Title, - Tooltip, + BarElement, CategoryScale, Chart as ChartJS, Legend, LinearScale, Title, Tooltip } from 'chart.js'; import { useEffect, useState } from 'react'; import { Bar } from 'react-chartjs-2'; @@ -71,7 +65,6 @@ export default function Analytics({ agentId }: AnalyticsProps) { string, { positive: number; negative: number } > | null>(null); - const [agent, setAgent] = useState(); const [messagesFilter, setMessagesFilter] = useState<{ label: string; value: string; @@ -97,21 +90,6 @@ export default function Analytics({ agentId }: AnalyticsProps) { const [loadingMessages, setLoadingMessages] = useLoaderState(true); const [loadingTokens, setLoadingTokens] = useLoaderState(true); const [loadingFeedback, setLoadingFeedback] = useLoaderState(true); - const [loadingAgent, setLoadingAgent] = useLoaderState(true); - - const fetchAgent = async (agentId: string) => { - setLoadingAgent(true); - try { - const response = await userService.getAgent(agentId ?? '', token); - if (!response.ok) throw new Error('Failed to fetch Chatbots'); - const agent = await response.json(); - setAgent(agent); - } catch (error) { - console.error(error); - } finally { - setLoadingAgent(false); - } - }; const fetchMessagesData = async (agent_id?: string, filter?: string) => { setLoadingMessages(true); @@ -174,27 +152,22 @@ export default function Analytics({ agentId }: AnalyticsProps) { }; useEffect(() => { - if (agentId) fetchAgent(agentId); - }, []); - - useEffect(() => { - const id = agent?.id; + const id = agentId; const filter = messagesFilter; fetchMessagesData(id, filter?.value); - }, [agent, messagesFilter]); + }, [agentId, messagesFilter]); useEffect(() => { - const id = agent?.id; + const id = agentId; const filter = tokenUsageFilter; fetchTokenData(id, filter?.value); - }, [agent, tokenUsageFilter]); + }, [agentId, tokenUsageFilter]); useEffect(() => { - const id = agent?.id; + const id = agentId; const filter = feedbackFilter; fetchFeedbackData(id, filter?.value); - }, [agent, feedbackFilter]); - + }, [agentId, feedbackFilter]); return (
{/* Messages Analytics */} diff --git a/frontend/src/settings/Logs.tsx b/frontend/src/settings/Logs.tsx index a14f5966..50d67b54 100644 --- a/frontend/src/settings/Logs.tsx +++ b/frontend/src/settings/Logs.tsx @@ -181,8 +181,7 @@ function Log({

From cc67d4a1e2034df2fb196d2b2d3e07beed8c5224 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 28 Apr 2025 17:49:29 +0100 Subject: [PATCH 3/5] process all request data implicitly --- application/worker.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/worker.py b/application/worker.py index e7ac85a9..bebd88a6 100755 --- a/application/worker.py +++ b/application/worker.py @@ -547,9 +547,7 @@ def agent_webhook_worker(self, agent_id, payload): agent_config = agents_collection.find_one({"_id": agent_oid}) if not agent_config: raise ValueError(f"Agent with ID {agent_id} not found.") - input_data = payload.get("query", "") - if input_data is None or not isinstance(input_data, str): - input_data = json.dumps(payload) + input_data = json.dumps(payload) except Exception as e: logging.error(f"Error processing agent webhook: {e}", exc_info=True) return {"status": "error", "error": str(e)} From 22c7015c695f2053492b2bc43c9c595bccd01411 Mon Sep 17 00:00:00 2001 From: Siddhant Rai Date: Tue, 29 Apr 2025 00:29:16 +0530 Subject: [PATCH 4/5] refactor: webhook listener handle both POST and GET requests --- application/api/user/routes.py | 192 +++++++++++++++++++++++---------- 1 file changed, 134 insertions(+), 58 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 528a4c29..f8e40b24 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -2,9 +2,10 @@ import datetime import json import math import os +import secrets import shutil import uuid -import secrets +from functools import wraps from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef @@ -18,8 +19,8 @@ from application.agents.tools.tool_manager import ToolManager from application.api.user.tasks import ( ingest, ingest_remote, - store_attachment, process_agent_webhook, + store_attachment, ) from application.core.mongo_db import MongoDB from application.core.settings import settings @@ -419,13 +420,14 @@ class UploadFile(Resource): user = secure_filename(decoded_token.get("sub")) job_name = secure_filename(request.form["name"]) - + try: from application.storage.storage_creator import StorageCreator + storage = StorageCreator.get_storage() - + base_path = f"{settings.UPLOAD_FOLDER}/{user}/{job_name}" - + if len(files) > 1: temp_files = [] for file in files: @@ -434,41 +436,56 @@ class UploadFile(Resource): storage.save_file(file, temp_path) temp_files.append(temp_path) print(f"Saved file: {filename}") - + zip_filename = f"{job_name}.zip" zip_path = f"{base_path}/{zip_filename}" - + def create_zip_archive(temp_paths, **kwargs): import tempfile + with tempfile.TemporaryDirectory() as temp_dir: for path in temp_paths: file_data = storage.get_file(path) - with open(os.path.join(temp_dir, os.path.basename(path)), 'wb') as f: + with open( + os.path.join(temp_dir, os.path.basename(path)), "wb" + ) as f: f.write(file_data.read()) - + # Create zip archive zip_temp = shutil.make_archive( base_name=os.path.join(temp_dir, job_name), format="zip", - root_dir=temp_dir + root_dir=temp_dir, ) - + return zip_temp - + zip_temp_path = create_zip_archive(temp_files) - with open(zip_temp_path, 'rb') as zip_file: + with open(zip_temp_path, "rb") as zip_file: storage.save_file(zip_file, zip_path) - + # Clean up temp files for temp_path in temp_files: storage.delete_file(temp_path) - + task = ingest.delay( settings.UPLOAD_FOLDER, [ - ".rst", ".md", ".pdf", ".txt", ".docx", ".csv", ".epub", - ".html", ".mdx", ".json", ".xlsx", ".pptx", ".png", - ".jpg", ".jpeg", + ".rst", + ".md", + ".pdf", + ".txt", + ".docx", + ".csv", + ".epub", + ".html", + ".mdx", + ".json", + ".xlsx", + ".pptx", + ".png", + ".jpg", + ".jpeg", ], job_name, zip_filename, @@ -479,15 +496,27 @@ class UploadFile(Resource): file = files[0] filename = secure_filename(file.filename) file_path = f"{base_path}/{filename}" - + storage.save_file(file, file_path) - + task = ingest.delay( settings.UPLOAD_FOLDER, [ - ".rst", ".md", ".pdf", ".txt", ".docx", ".csv", ".epub", - ".html", ".mdx", ".json", ".xlsx", ".pptx", ".png", - ".jpg", ".jpeg", + ".rst", + ".md", + ".pdf", + ".txt", + ".docx", + ".csv", + ".epub", + ".html", + ".mdx", + ".json", + ".xlsx", + ".pptx", + ".png", + ".jpg", + ".jpeg", ], job_name, filename, @@ -497,7 +526,7 @@ class UploadFile(Resource): except Exception as err: current_app.logger.error(f"Error uploading file: {err}") return make_response(jsonify({"success": False}), 400) - + return make_response(jsonify({"success": True, "task_id": task.id}), 200) @@ -1386,39 +1415,88 @@ class AgentWebhook(Resource): ) -@user_ns.route(f"/api/webhooks/agents/") -class AgentWebhookListener(Resource): - @api.doc(description="Webhook listener for agent events") - def post(self, webhook_token): +def require_agent(func): + @wraps(func) + def wrapper(*args, **kwargs): + webhook_token = kwargs.get("webhook_token") + if not webhook_token: + return make_response( + jsonify({"success": False, "message": "Webhook token missing"}), 400 + ) + agent = agents_collection.find_one( {"incoming_webhook_token": webhook_token}, {"_id": 1} ) if not agent: + current_app.logger.warning( + f"Webhook attempt with invalid token: {webhook_token}" + ) return make_response( jsonify({"success": False, "message": "Agent not found"}), 404 ) - data = request.get_json() - if not data: - return make_response( - jsonify({"success": False, "message": "No data provided"}), 400 + + kwargs["agent"] = agent + kwargs["agent_id_str"] = str(agent["_id"]) + return func(*args, **kwargs) + + return wrapper + + +@user_ns.route(f"/api/webhooks/agents/") +class AgentWebhookListener(Resource): + method_decorators = [require_agent] + + def _enqueue_webhook_task(self, agent_id_str, payload, source_method): + if not payload: + current_app.logger.warning( + f"Webhook ({source_method}) received for agent {agent_id_str} with empty payload." ) - agent_id_str = str(agent["_id"]) current_app.logger.info( - f"Incoming webhook received for agent {agent_id_str}. Enqueuing task." + f"Incoming {source_method} webhook for agent {agent_id_str}. Enqueuing task with payload: {payload}" ) try: task = process_agent_webhook.delay( agent_id=agent_id_str, - payload=data, + payload=payload, ) + current_app.logger.info( + f"Task {task.id} enqueued for agent {agent_id_str} ({source_method})." + ) + return make_response(jsonify({"success": True, "task_id": task.id}), 200) except Exception as err: - current_app.logger.error(f"Error processing webhook: {err}") - return make_response( - jsonify({"success": False, "message": "Error processing webhook"}), 400 + current_app.logger.error( + f"Error enqueuing webhook task ({source_method}) for agent {agent_id_str}: {err}", + exc_info=True, ) - return make_response(jsonify({"success": True, "task_id": task.id}), 200) + return make_response( + jsonify({"success": False, "message": "Error processing webhook"}), 500 + ) + + @api.doc( + description="Webhook listener for agent events (POST). Expects JSON payload, which is used to trigger processing.", + ) + def post(self, webhook_token, agent, agent_id_str): + payload = request.get_json() + if payload is None: + return make_response( + jsonify( + { + "success": False, + "message": "Invalid or missing JSON data in request body", + } + ), + 400, + ) + return self._enqueue_webhook_task(agent_id_str, payload, source_method="POST") + + @api.doc( + description="Webhook listener for agent events (GET). Uses URL query parameters as payload to trigger processing.", + ) + def get(self, webhook_token, agent, agent_id_str): + payload = request.args.to_dict(flat=True) + return self._enqueue_webhook_task(agent_id_str, payload, source_method="GET") @user_ns.route("/api/share") @@ -2872,9 +2950,9 @@ class StoreAttachment(Resource): decoded_token = request.decoded_token if not decoded_token: return make_response(jsonify({"success": False}), 401) - + file = request.files.get("file") - + if not file or file.filename == "": return make_response( jsonify({"status": "error", "message": "Missing file"}), @@ -2882,35 +2960,33 @@ class StoreAttachment(Resource): ) user = secure_filename(decoded_token.get("sub")) - + try: attachment_id = ObjectId() original_filename = secure_filename(file.filename) relative_path = f"{settings.UPLOAD_FOLDER}/{user}/attachments/{str(attachment_id)}/{original_filename}" - + file_content = file.read() - + file_info = { "filename": original_filename, "attachment_id": str(attachment_id), "path": relative_path, - "file_content": file_content + "file_content": file_content, } - - task = store_attachment.delay( - file_info, - user - ) - + + task = store_attachment.delay(file_info, user) + return make_response( - jsonify({ - "success": True, - "task_id": task.id, - "message": "File uploaded successfully. Processing started." - }), - 200 + jsonify( + { + "success": True, + "task_id": task.id, + "message": "File uploaded successfully. Processing started.", + } + ), + 200, ) except Exception as err: current_app.logger.error(f"Error storing attachment: {err}") return make_response(jsonify({"success": False, "error": str(err)}), 400) - From 330276cdf7363f4b9b0c2e0edb7dfa0b52cf73a7 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 28 Apr 2025 22:32:13 +0100 Subject: [PATCH 5/5] fix: lint for ruff --- application/api/user/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index f8e40b24..d96d6202 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -1442,7 +1442,7 @@ def require_agent(func): return wrapper -@user_ns.route(f"/api/webhooks/agents/") +@user_ns.route("/api/webhooks/agents/") class AgentWebhookListener(Resource): method_decorators = [require_agent]