diff --git a/application/agents/base.py b/application/agents/base.py index e4b76ca1..d44244cf 100644 --- a/application/agents/base.py +++ b/application/agents/base.py @@ -256,7 +256,7 @@ class BaseAgent(ABC): model=self.gpt_model, messages=messages, tools=self.tools ) if log_context: - data = build_stack_data(self.llm) + data = build_stack_data(self.llm, exclude_attributes=["client"]) log_context.stacks.append({"component": "llm", "data": data}) return resp @@ -272,6 +272,6 @@ class BaseAgent(ABC): self, resp, tools_dict, messages, attachments ) if log_context: - data = build_stack_data(self.llm_handler) + data = build_stack_data(self.llm_handler, exclude_attributes=["tool_calls"]) log_context.stacks.append({"component": "llm_handler", "data": data}) return resp diff --git a/application/agents/classic_agent.py b/application/agents/classic_agent.py index bf472cd0..b96a77fc 100644 --- a/application/agents/classic_agent.py +++ b/application/agents/classic_agent.py @@ -48,15 +48,13 @@ class ClassicAgent(BaseAgent): ): yield {"answer": resp.message.content} else: - # completion = self.llm.gen_stream( - # model=self.gpt_model, messages=messages, tools=self.tools - # ) - # log type of resp - logger.info(f"Response type: {type(resp)}") - logger.info(f"Response: {resp}") for line in resp: if isinstance(line, str): yield {"answer": line} + log_context.stacks.append( + {"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}} + ) + yield {"sources": retrieved_data} yield {"tool_calls": self.tool_calls.copy()} diff --git a/application/agents/react_agent.py b/application/agents/react_agent.py index 3fae1fda..a5d47850 100644 --- a/application/agents/react_agent.py +++ b/application/agents/react_agent.py @@ -82,6 +82,10 @@ class ReActAgent(BaseAgent): if isinstance(line, str): self.observations.append(line) + log_context.stacks.append( + {"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}} + ) + yield {"sources": retrieved_data} yield {"tool_calls": self.tool_calls.copy()} diff --git a/application/api/user/routes.py b/application/api/user/routes.py index d9c41c8f..d96d6202 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -2,8 +2,10 @@ import datetime import json import math import os +import secrets import shutil import uuid +from functools import wraps from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef @@ -14,7 +16,12 @@ from werkzeug.utils import secure_filename from application.agents.tools.tool_manager import ToolManager -from application.api.user.tasks import ingest, ingest_remote, store_attachment +from application.api.user.tasks import ( + ingest, + ingest_remote, + process_agent_webhook, + store_attachment, +) from application.core.mongo_db import MongoDB from application.core.settings import settings from application.extensions import api @@ -413,13 +420,14 @@ class UploadFile(Resource): user = secure_filename(decoded_token.get("sub")) job_name = secure_filename(request.form["name"]) - + try: from application.storage.storage_creator import StorageCreator + storage = StorageCreator.get_storage() - + base_path = f"{settings.UPLOAD_FOLDER}/{user}/{job_name}" - + if len(files) > 1: temp_files = [] for file in files: @@ -428,41 +436,56 @@ class UploadFile(Resource): storage.save_file(file, temp_path) temp_files.append(temp_path) print(f"Saved file: {filename}") - + zip_filename = f"{job_name}.zip" zip_path = f"{base_path}/{zip_filename}" - + def create_zip_archive(temp_paths, **kwargs): import tempfile + with tempfile.TemporaryDirectory() as temp_dir: for path in temp_paths: file_data = storage.get_file(path) - with open(os.path.join(temp_dir, os.path.basename(path)), 'wb') as f: + with open( + os.path.join(temp_dir, os.path.basename(path)), "wb" + ) as f: f.write(file_data.read()) - + # Create zip archive zip_temp = shutil.make_archive( base_name=os.path.join(temp_dir, job_name), format="zip", - root_dir=temp_dir + root_dir=temp_dir, ) - + return zip_temp - + zip_temp_path = create_zip_archive(temp_files) - with open(zip_temp_path, 'rb') as zip_file: + with open(zip_temp_path, "rb") as zip_file: storage.save_file(zip_file, zip_path) - + # Clean up temp files for temp_path in temp_files: storage.delete_file(temp_path) - + task = ingest.delay( settings.UPLOAD_FOLDER, [ - ".rst", ".md", ".pdf", ".txt", ".docx", ".csv", ".epub", - ".html", ".mdx", ".json", ".xlsx", ".pptx", ".png", - ".jpg", ".jpeg", + ".rst", + ".md", + ".pdf", + ".txt", + ".docx", + ".csv", + ".epub", + ".html", + ".mdx", + ".json", + ".xlsx", + ".pptx", + ".png", + ".jpg", + ".jpeg", ], job_name, zip_filename, @@ -473,15 +496,27 @@ class UploadFile(Resource): file = files[0] filename = secure_filename(file.filename) file_path = f"{base_path}/{filename}" - + storage.save_file(file, file_path) - + task = ingest.delay( settings.UPLOAD_FOLDER, [ - ".rst", ".md", ".pdf", ".txt", ".docx", ".csv", ".epub", - ".html", ".mdx", ".json", ".xlsx", ".pptx", ".png", - ".jpg", ".jpeg", + ".rst", + ".md", + ".pdf", + ".txt", + ".docx", + ".csv", + ".epub", + ".html", + ".mdx", + ".json", + ".xlsx", + ".pptx", + ".png", + ".jpg", + ".jpeg", ], job_name, filename, @@ -491,7 +526,7 @@ class UploadFile(Resource): except Exception as err: current_app.logger.error(f"Error uploading file: {err}") return make_response(jsonify({"success": False}), 400) - + return make_response(jsonify({"success": True, "task_id": task.id}), 200) @@ -1333,6 +1368,137 @@ class DeleteAgent(Resource): return make_response(jsonify({"id": deleted_id}), 200) +@user_ns.route("/api/agent_webhook") +class AgentWebhook(Resource): + @api.doc( + params={"id": "ID of the agent"}, + description="Generate webhook URL for the agent", + ) + def get(self): + decoded_token = request.decoded_token + if not decoded_token: + return make_response(jsonify({"success": False}), 401) + user = decoded_token.get("sub") + agent_id = request.args.get("id") + if not agent_id: + return make_response( + jsonify({"success": False, "message": "ID is required"}), 400 + ) + + try: + agent = agents_collection.find_one( + {"_id": ObjectId(agent_id), "user": user} + ) + if not agent: + return make_response( + jsonify({"success": False, "message": "Agent not found"}), 404 + ) + + webhook_token = agent.get("incoming_webhook_token") + if not webhook_token: + webhook_token = secrets.token_urlsafe(32) + agents_collection.update_one( + {"_id": ObjectId(agent_id), "user": user}, + {"$set": {"incoming_webhook_token": webhook_token}}, + ) + base_url = settings.API_URL.rstrip("/") + full_webhook_url = f"{base_url}/api/webhooks/agents/{webhook_token}" + + except Exception as err: + current_app.logger.error(f"Error generating webhook URL: {err}") + return make_response( + jsonify({"success": False, "message": "Error generating webhook URL"}), + 400, + ) + return make_response( + jsonify({"success": True, "webhook_url": full_webhook_url}), 200 + ) + + +def require_agent(func): + @wraps(func) + def wrapper(*args, **kwargs): + webhook_token = kwargs.get("webhook_token") + if not webhook_token: + return make_response( + jsonify({"success": False, "message": "Webhook token missing"}), 400 + ) + + agent = agents_collection.find_one( + {"incoming_webhook_token": webhook_token}, {"_id": 1} + ) + if not agent: + current_app.logger.warning( + f"Webhook attempt with invalid token: {webhook_token}" + ) + return make_response( + jsonify({"success": False, "message": "Agent not found"}), 404 + ) + + kwargs["agent"] = agent + kwargs["agent_id_str"] = str(agent["_id"]) + return func(*args, **kwargs) + + return wrapper + + +@user_ns.route("/api/webhooks/agents/") +class AgentWebhookListener(Resource): + method_decorators = [require_agent] + + def _enqueue_webhook_task(self, agent_id_str, payload, source_method): + if not payload: + current_app.logger.warning( + f"Webhook ({source_method}) received for agent {agent_id_str} with empty payload." + ) + + current_app.logger.info( + f"Incoming {source_method} webhook for agent {agent_id_str}. Enqueuing task with payload: {payload}" + ) + + try: + task = process_agent_webhook.delay( + agent_id=agent_id_str, + payload=payload, + ) + current_app.logger.info( + f"Task {task.id} enqueued for agent {agent_id_str} ({source_method})." + ) + return make_response(jsonify({"success": True, "task_id": task.id}), 200) + except Exception as err: + current_app.logger.error( + f"Error enqueuing webhook task ({source_method}) for agent {agent_id_str}: {err}", + exc_info=True, + ) + return make_response( + jsonify({"success": False, "message": "Error processing webhook"}), 500 + ) + + @api.doc( + description="Webhook listener for agent events (POST). Expects JSON payload, which is used to trigger processing.", + ) + def post(self, webhook_token, agent, agent_id_str): + payload = request.get_json() + if payload is None: + return make_response( + jsonify( + { + "success": False, + "message": "Invalid or missing JSON data in request body", + } + ), + 400, + ) + return self._enqueue_webhook_task(agent_id_str, payload, source_method="POST") + + @api.doc( + description="Webhook listener for agent events (GET). Uses URL query parameters as payload to trigger processing.", + ) + def get(self, webhook_token, agent, agent_id_str): + payload = request.args.to_dict(flat=True) + return self._enqueue_webhook_task(agent_id_str, payload, source_method="GET") + + @user_ns.route("/api/share") class ShareConversation(Resource): share_conversation_model = api.model( @@ -2784,9 +2950,9 @@ class StoreAttachment(Resource): decoded_token = request.decoded_token if not decoded_token: return make_response(jsonify({"success": False}), 401) - + file = request.files.get("file") - + if not file or file.filename == "": return make_response( jsonify({"status": "error", "message": "Missing file"}), @@ -2794,35 +2960,33 @@ class StoreAttachment(Resource): ) user = secure_filename(decoded_token.get("sub")) - + try: attachment_id = ObjectId() original_filename = secure_filename(file.filename) relative_path = f"{settings.UPLOAD_FOLDER}/{user}/attachments/{str(attachment_id)}/{original_filename}" - + file_content = file.read() - + file_info = { "filename": original_filename, "attachment_id": str(attachment_id), "path": relative_path, - "file_content": file_content + "file_content": file_content, } - - task = store_attachment.delay( - file_info, - user - ) - + + task = store_attachment.delay(file_info, user) + return make_response( - jsonify({ - "success": True, - "task_id": task.id, - "message": "File uploaded successfully. Processing started." - }), - 200 + jsonify( + { + "success": True, + "task_id": task.id, + "message": "File uploaded successfully. Processing started.", + } + ), + 200, ) except Exception as err: current_app.logger.error(f"Error storing attachment: {err}") return make_response(jsonify({"success": False, "error": str(err)}), 400) - diff --git a/application/api/user/tasks.py b/application/api/user/tasks.py index c9d4d39d..fffa9ba9 100644 --- a/application/api/user/tasks.py +++ b/application/api/user/tasks.py @@ -1,7 +1,13 @@ from datetime import timedelta from application.celery_init import celery -from application.worker import ingest_worker, remote_worker, sync_worker, attachment_worker +from application.worker import ( + agent_webhook_worker, + attachment_worker, + ingest_worker, + remote_worker, + sync_worker, +) @celery.task(bind=True) @@ -28,6 +34,12 @@ def store_attachment(self, file_info, user): return resp +@celery.task(bind=True) +def process_agent_webhook(self, agent_id, payload): + resp = agent_webhook_worker(self, agent_id, payload) + return resp + + @celery.on_after_configure.connect def setup_periodic_tasks(sender, **kwargs): sender.add_periodic_task( diff --git a/application/logging.py b/application/logging.py index ed07f858..b447ffa8 100644 --- a/application/logging.py +++ b/application/logging.py @@ -30,6 +30,8 @@ def build_stack_data( exclude_attributes: List[str] = None, custom_data: Dict = None, ) -> Dict: + if obj is None: + raise ValueError("The 'obj' parameter cannot be None") data = {} if include_attributes is None: include_attributes = [] @@ -57,8 +59,8 @@ def build_stack_data( data[attr_name] = [str(item) for item in attr_value] elif isinstance(attr_value, dict): data[attr_name] = {k: str(v) for k, v in attr_value.items()} - else: - data[attr_name] = str(attr_value) + except AttributeError as e: + logging.warning(f"AttributeError while accessing {attr_name}: {e}") except AttributeError: pass if custom_data: diff --git a/application/worker.py b/application/worker.py index 3f542b6a..bebd88a6 100755 --- a/application/worker.py +++ b/application/worker.py @@ -1,29 +1,35 @@ +import datetime +import io +import json import logging +import mimetypes import os import shutil import string -import zipfile -import io -import datetime -import mimetypes -import requests import tempfile +import zipfile from collections import Counter from urllib.parse import urljoin -from application.storage.storage_creator import StorageCreator -from application.utils import num_tokens_from_string -from application.core.settings import settings -from application.parser.file.bulk import SimpleDirectoryReader +import requests +from bson.dbref import DBRef from bson.objectid import ObjectId +from application.agents.agent_creator import AgentCreator +from application.api.answer.routes import get_prompt + from application.core.mongo_db import MongoDB +from application.core.settings import settings +from application.parser.chunking import Chunker from application.parser.embedding_pipeline import embed_and_store_documents +from application.parser.file.bulk import SimpleDirectoryReader from application.parser.remote.remote_creator import RemoteCreator from application.parser.schema.base import Document -from application.parser.chunking import Chunker -from application.utils import count_tokens_docs +from application.retriever.retriever_creator import RetrieverCreator + +from application.storage.storage_creator import StorageCreator +from application.utils import count_tokens_docs, num_tokens_from_string mongo = MongoDB.get_client() db = mongo[settings.MONGO_DB_NAME] @@ -34,18 +40,22 @@ MIN_TOKENS = 150 MAX_TOKENS = 1250 RECURSION_DEPTH = 2 + # Define a function to extract metadata from a given filename. def metadata_from_filename(title): return {"title": title} + # Define a function to generate a random string of a given length. def generate_random_string(length): return "".join([string.ascii_letters[i % 52] for i in range(length)]) + current_dir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ) + def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5): """ Recursively extract zip files with a limit on recursion depth. @@ -76,6 +86,7 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5): file_path = os.path.join(root, file) extract_zip_recursive(file_path, root, current_depth + 1, max_depth) + def download_file(url, params, dest_path): try: response = requests.get(url, params=params) @@ -86,6 +97,7 @@ def download_file(url, params, dest_path): logging.error(f"Error downloading file: {e}") raise + def upload_index(full_path, file_data): try: if settings.VECTOR_STORE == "faiss": @@ -94,7 +106,9 @@ def upload_index(full_path, file_data): "file_pkl": open(full_path + "/index.pkl", "rb"), } response = requests.post( - urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data + urljoin(settings.API_URL, "/api/upload_index"), + files=files, + data=file_data, ) else: response = requests.post( @@ -109,6 +123,76 @@ def upload_index(full_path, file_data): for file in files.values(): file.close() + +def run_agent_logic(agent_config, input_data): + try: + source = agent_config.get("source") + retriever = agent_config.get("retriever", "classic") + if isinstance(source, DBRef): + source_doc = db.dereference(source) + source = str(source_doc["_id"]) + retriever = source_doc.get("retriever", agent_config.get("retriever")) + else: + source = {} + source = {"active_docs": source} + chunks = int(agent_config.get("chunks", 2)) + prompt_id = agent_config.get("prompt_id", "default") + user_api_key = agent_config["key"] + agent_type = agent_config.get("agent_type", "classic") + decoded_token = {"sub": agent_config.get("user")} + prompt = get_prompt(prompt_id) + agent = AgentCreator.create_agent( + agent_type, + endpoint="webhook", + llm_name=settings.LLM_NAME, + gpt_model=settings.MODEL_NAME, + api_key=settings.API_KEY, + user_api_key=user_api_key, + prompt=prompt, + chat_history=[], + decoded_token=decoded_token, + attachments=[], + ) + retriever = RetrieverCreator.create_retriever( + retriever, + source=source, + chat_history=[], + prompt=prompt, + chunks=chunks, + token_limit=settings.DEFAULT_MAX_HISTORY, + gpt_model=settings.MODEL_NAME, + user_api_key=user_api_key, + decoded_token=decoded_token, + ) + answer = agent.gen(query=input_data, retriever=retriever) + response_full = "" + thought = "" + source_log_docs = [] + tool_calls = [] + + for line in answer: + if "answer" in line: + response_full += str(line["answer"]) + elif "sources" in line: + source_log_docs.extend(line["sources"]) + elif "tool_calls" in line: + tool_calls.extend(line["tool_calls"]) + elif "thought" in line: + thought += line["thought"] + + result = { + "answer": response_full, + "sources": source_log_docs, + "tool_calls": tool_calls, + "thought": thought, + } + logging.info(f"Agent response: {result}") + return result + except Exception as e: + logging.error(f"Error in run_agent_logic: {e}", exc_info=True) + raise + + # Define the main function for ingesting and processing documents. def ingest_worker( self, directory, formats, name_job, filename, user, retriever="classic" @@ -133,36 +217,33 @@ def ingest_worker( limit = None exclude = True sample = False - + storage = StorageCreator.get_storage() - + full_path = os.path.join(directory, user, name_job) source_file_path = os.path.join(full_path, filename) - + logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job}) - + # Create temporary working directory with tempfile.TemporaryDirectory() as temp_dir: try: os.makedirs(temp_dir, exist_ok=True) - + # Download file from storage to temp directory temp_file_path = os.path.join(temp_dir, filename) file_data = storage.get_file(source_file_path) - - with open(temp_file_path, 'wb') as f: + + with open(temp_file_path, "wb") as f: f.write(file_data.read()) - + self.update_state(state="PROGRESS", meta={"current": 1}) # Handle zip files - if filename.endswith('.zip'): + if filename.endswith(".zip"): logging.info(f"Extracting zip file: {filename}") extract_zip_recursive( - temp_file_path, - temp_dir, - current_depth=0, - max_depth=RECURSION_DEPTH + temp_file_path, temp_dir, current_depth=0, max_depth=RECURSION_DEPTH ) if sample: @@ -182,25 +263,25 @@ def ingest_worker( chunking_strategy="classic_chunk", max_tokens=MAX_TOKENS, min_tokens=MIN_TOKENS, - duplicate_headers=False + duplicate_headers=False, ) raw_docs = chunker.chunk(documents=raw_docs) - + docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs] - + id = ObjectId() - - vector_store_path = os.path.join(temp_dir, 'vector_store') + + vector_store_path = os.path.join(temp_dir, "vector_store") os.makedirs(vector_store_path, exist_ok=True) - + embed_and_store_documents(docs, vector_store_path, id, self) - + tokens = count_tokens_docs(docs) - + self.update_state(state="PROGRESS", meta={"current": 100}) if sample: - for i in range(min(5, len(raw_docs))): + for i in range(min(5, len(raw_docs))): logging.info(f"Sample document {i}: {raw_docs[i]}") file_data = { "name": name_job, @@ -212,7 +293,6 @@ def ingest_worker( "type": "local", } - upload_index(vector_store_path, file_data) except Exception as e: @@ -228,6 +308,7 @@ def ingest_worker( "limited": False, } + def remote_worker( self, source_data, @@ -254,7 +335,7 @@ def remote_worker( chunking_strategy="classic_chunk", max_tokens=MAX_TOKENS, min_tokens=MIN_TOKENS, - duplicate_headers=False + duplicate_headers=False, ) docs = chunker.chunk(documents=raw_docs) docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs] @@ -296,6 +377,7 @@ def remote_worker( logging.info("remote_worker task completed successfully") return {"urls": source_data, "name_job": name_job, "user": user, "limited": False} + def sync( self, source_data, @@ -325,6 +407,7 @@ def sync( return {"status": "error", "error": str(e)} return {"status": "success"} + def sync_worker(self, frequency): sync_counts = Counter() sources = sources_collection.find() @@ -368,15 +451,17 @@ def attachment_worker(self, file_info, user): self.update_state(state="PROGRESS", meta={"current": 10}) storage_type = getattr(settings, "STORAGE_TYPE", "local") storage = StorageCreator.create_storage(storage_type) - self.update_state(state="PROGRESS", meta={"current": 30, "status": "Processing content"}) + self.update_state( + state="PROGRESS", meta={"current": 30, "status": "Processing content"} + ) - with tempfile.NamedTemporaryFile(suffix=os.path.splitext(filename)[1]) as temp_file: + with tempfile.NamedTemporaryFile( + suffix=os.path.splitext(filename)[1] + ) as temp_file: temp_file.write(file_content) temp_file.flush() reader = SimpleDirectoryReader( - input_files=[temp_file.name], - exclude_hidden=True, - errors="ignore" + input_files=[temp_file.name], exclude_hidden=True, errors="ignore" ) documents = reader.load_data() @@ -387,31 +472,40 @@ def attachment_worker(self, file_info, user): content = documents[0].text token_count = num_tokens_from_string(content) - self.update_state(state="PROGRESS", meta={"current": 60, "status": "Saving file"}) + self.update_state( + state="PROGRESS", meta={"current": 60, "status": "Saving file"} + ) file_obj = io.BytesIO(file_content) metadata = storage.save_file(file_obj, relative_path) - mime_type = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream" - self.update_state(state="PROGRESS", meta={"current": 80, "status": "Storing in database"}) + self.update_state( + state="PROGRESS", meta={"current": 80, "status": "Storing in database"} + ) doc_id = ObjectId(attachment_id) - attachments_collection.insert_one({ - "_id": doc_id, - "user": user, - "path": relative_path, - "content": content, - "token_count": token_count, - "mime_type": mime_type, - "date": datetime.datetime.now(), - "metadata": metadata - }) + attachments_collection.insert_one( + { + "_id": doc_id, + "user": user, + "path": relative_path, + "content": content, + "token_count": token_count, + "mime_type": mime_type, + "date": datetime.datetime.now(), + "metadata": metadata, + } + ) - logging.info(f"Stored attachment with ID: {attachment_id}", - extra={"user": user}) + logging.info( + f"Stored attachment with ID: {attachment_id}", extra={"user": user} + ) - self.update_state(state="PROGRESS", meta={"current": 100, "status": "Complete"}) + self.update_state( + state="PROGRESS", meta={"current": 100, "status": "Complete"} + ) return { "filename": filename, @@ -419,9 +513,54 @@ def attachment_worker(self, file_info, user): "token_count": token_count, "attachment_id": attachment_id, "mime_type": mime_type, - "metadata": metadata + "metadata": metadata, } except Exception as e: - logging.error(f"Error processing file {filename}: {e}", extra={"user": user}, exc_info=True) + logging.error( + f"Error processing file {filename}: {e}", + extra={"user": user}, + exc_info=True, + ) raise + + +def agent_webhook_worker(self, agent_id, payload): + """ + Process the webhook payload for an agent. + + Args: + self: Reference to the instance of the task. + agent_id (str): Unique identifier for the agent. + payload (dict): The payload data from the webhook. + + Returns: + dict: Information about the processed webhook. + """ + mongo = MongoDB.get_client() + db = mongo["docsgpt"] + agents_collection = db["agents"] + + self.update_state(state="PROGRESS", meta={"current": 1}) + try: + agent_oid = ObjectId(agent_id) + agent_config = agents_collection.find_one({"_id": agent_oid}) + if not agent_config: + raise ValueError(f"Agent with ID {agent_id} not found.") + input_data = json.dumps(payload) + except Exception as e: + logging.error(f"Error processing agent webhook: {e}", exc_info=True) + return {"status": "error", "error": str(e)} + + self.update_state(state="PROGRESS", meta={"current": 50}) + try: + result = run_agent_logic(agent_config, input_data) + except Exception as e: + logging.error(f"Error running agent logic: {e}", exc_info=True) + return {"status": "error", "error": str(e)} + finally: + self.update_state(state="PROGRESS", meta={"current": 100}) + logging.info( + f"Webhook processed for agent {agent_id}", extra={"agent_id": agent_id} + ) + return {"status": "success", "result": result} diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 043bbf58..fa250e66 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@reduxjs/toolkit": "^2.5.1", "chart.js": "^4.4.4", + "clsx": "^2.1.1", "i18next": "^24.2.0", "i18next-browser-languagedetector": "^8.0.2", "prop-types": "^15.8.1", @@ -2751,6 +2752,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", @@ -9405,6 +9415,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, "node_modules/unified": { "version": "11.0.5", "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", diff --git a/frontend/package.json b/frontend/package.json index 45058e98..62afaad3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,6 +21,7 @@ "dependencies": { "@reduxjs/toolkit": "^2.5.1", "chart.js": "^4.4.4", + "clsx": "^2.1.1", "i18next": "^24.2.0", "i18next-browser-languagedetector": "^8.0.2", "prop-types": "^15.8.1", diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index 0e357a6d..53487dd6 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -44,6 +44,7 @@ import { setModalStateDeleteConv, setSelectedAgent, setAgents, + selectAgents, } from './preferences/preferenceSlice'; import Upload from './upload/Upload'; @@ -63,6 +64,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { const conversations = useSelector(selectConversations); const conversationId = useSelector(selectConversationId); const modalStateDeleteConv = useSelector(selectModalStateDeleteConv); + const agents = useSelector(selectAgents); const selectedAgent = useSelector(selectSelectedAgent); const { isMobile } = useMediaQuery(); @@ -76,6 +78,31 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { const navRef = useRef(null); + async function fetchRecentAgents() { + try { + let recentAgents: Agent[] = []; + if (!agents) { + const response = await userService.getAgents(token); + if (!response.ok) throw new Error('Failed to fetch agents'); + const data: Agent[] = await response.json(); + dispatch(setAgents(data)); + recentAgents = data; + } else recentAgents = agents; + setRecentAgents( + recentAgents + .filter((agent: Agent) => agent.status === 'published') + .sort( + (a: Agent, b: Agent) => + new Date(b.last_used_at ?? 0).getTime() - + new Date(a.last_used_at ?? 0).getTime(), + ) + .slice(0, 3), + ); + } catch (error) { + console.error('Failed to fetch recent agents: ', error); + } + } + async function fetchConversations() { dispatch(setConversations({ ...conversations, loading: true })); return await getConversations(token) @@ -88,25 +115,11 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { }); } - async function getAgents() { - const response = await userService.getAgents(token); - if (!response.ok) throw new Error('Failed to fetch agents'); - const data: Agent[] = await response.json(); - dispatch(setAgents(data)); - setRecentAgents( - data - .filter((agent: Agent) => agent.status === 'published') - .sort( - (a: Agent, b: Agent) => - new Date(b.last_used_at ?? 0).getTime() - - new Date(a.last_used_at ?? 0).getTime(), - ) - .slice(0, 3), - ); - } + useEffect(() => { + if (token) fetchRecentAgents(); + }, [agents, token, dispatch]); useEffect(() => { - if (recentAgents.length === 0) getAgents(); if (!conversations?.data) fetchConversations(); if (queries.length === 0) resetConversation(); }, [conversations?.data, dispatch]); diff --git a/frontend/src/agents/AgentLogs.tsx b/frontend/src/agents/AgentLogs.tsx index 3773e54f..864a85fe 100644 --- a/frontend/src/agents/AgentLogs.tsx +++ b/frontend/src/agents/AgentLogs.tsx @@ -1,12 +1,40 @@ +import { useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; import { useNavigate, useParams } from 'react-router-dom'; +import userService from '../api/services/userService'; import ArrowLeft from '../assets/arrow-left.svg'; +import { selectToken } from '../preferences/preferenceSlice'; import Analytics from '../settings/Analytics'; import Logs from '../settings/Logs'; +import Spinner from '../components/Spinner'; +import { Agent } from './types'; export default function AgentLogs() { const navigate = useNavigate(); const { agentId } = useParams(); + const token = useSelector(selectToken); + + const [agent, setAgent] = useState(); + const [loadingAgent, setLoadingAgent] = useState(true); + + const fetchAgent = async (agentId: string) => { + setLoadingAgent(true); + try { + const response = await userService.getAgent(agentId ?? '', token); + if (!response.ok) throw new Error('Failed to fetch Chatbots'); + const agent = await response.json(); + setAgent(agent); + } catch (error) { + console.error(error); + } finally { + setLoadingAgent(false); + } + }; + + useEffect(() => { + if (agentId) fetchAgent(agentId); + }, [agentId, token]); return (
@@ -25,8 +53,29 @@ export default function AgentLogs() { Agent Logs
- - +
+

+ Agent Name +

+ {agent && ( +

{agent.name}

+ )} +
+ {loadingAgent ? ( +
+ +
+ ) : ( + agent && + )} + {loadingAgent ? ( +
+ {' '} + +
+ ) : ( + agent && + )}
); } diff --git a/frontend/src/agents/AgentPreview.tsx b/frontend/src/agents/AgentPreview.tsx index 5eaf10a9..621ac477 100644 --- a/frontend/src/agents/AgentPreview.tsx +++ b/frontend/src/agents/AgentPreview.tsx @@ -141,6 +141,7 @@ export default function AgentPreview() { loading={status === 'loading'} showSourceButton={selectedAgent ? false : true} showToolButton={selectedAgent ? false : true} + autoFocus={false} />

This is a preview of the agent. You can publish it to start using it diff --git a/frontend/src/agents/NewAgent.tsx b/frontend/src/agents/NewAgent.tsx index 37466a86..0cccfbe6 100644 --- a/frontend/src/agents/NewAgent.tsx +++ b/frontend/src/agents/NewAgent.tsx @@ -11,10 +11,7 @@ import AgentDetailsModal from '../modals/AgentDetailsModal'; import ConfirmationModal from '../modals/ConfirmationModal'; import { ActiveState, Doc, Prompt } from '../models/misc'; import { - selectSelectedAgent, - selectSourceDocs, - selectToken, - setSelectedAgent, + selectSelectedAgent, selectSourceDocs, selectToken, setSelectedAgent } from '../preferences/preferenceSlice'; import PromptsModal from '../preferences/PromptsModal'; import { UserToolType } from '../settings/types'; @@ -155,9 +152,10 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { const data = await response.json(); if (data.id) setAgent((prev) => ({ ...prev, id: data.id })); if (data.key) setAgent((prev) => ({ ...prev, key: data.key })); - if (effectiveMode === 'new') { - setAgentDetails('ACTIVE'); + if (effectiveMode === 'new' || effectiveMode === 'draft') { setEffectiveMode('edit'); + setAgent((prev) => ({ ...prev, status: 'published' })); + setAgentDetails('ACTIVE'); } }; @@ -286,9 +284,10 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { )} {modeConfig[effectiveMode].showAccessDetails && ( )} @@ -408,7 +407,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { agent.prompt_id ? prompts.filter( (prompt) => prompt.id === agent.prompt_id, - )[0].name || null + )[0]?.name || null : null } onSelect={(option: { label: string; value: string }) => @@ -532,7 +531,7 @@ function AgentPreviewArea() { const selectedAgent = useSelector(selectSelectedAgent); return (

- {selectedAgent?.id ? ( + {selectedAgent?.status === 'published' ? (
@@ -540,7 +539,7 @@ function AgentPreviewArea() {
{' '}

- Published agents can be previewd here + Published agents can be previewed here

)} diff --git a/frontend/src/agents/index.tsx b/frontend/src/agents/index.tsx index 49123cd6..c2edb34a 100644 --- a/frontend/src/agents/index.tsx +++ b/frontend/src/agents/index.tsx @@ -12,7 +12,13 @@ import ThreeDots from '../assets/three-dots.svg'; import ContextMenu, { MenuOption } from '../components/ContextMenu'; import ConfirmationModal from '../modals/ConfirmationModal'; import { ActiveState } from '../models/misc'; -import { selectToken, setSelectedAgent } from '../preferences/preferenceSlice'; +import { + selectToken, + setSelectedAgent, + setAgents, + selectAgents, + selectSelectedAgent, +} from '../preferences/preferenceSlice'; import AgentLogs from './AgentLogs'; import NewAgent from './NewAgent'; import { Agent } from './types'; @@ -31,9 +37,12 @@ export default function Agents() { function AgentsList() { const navigate = useNavigate(); + const dispatch = useDispatch(); const token = useSelector(selectToken); + const agents = useSelector(selectAgents); + const selectedAgent = useSelector(selectSelectedAgent); - const [userAgents, setUserAgents] = useState([]); + const [userAgents, setUserAgents] = useState(agents || []); const [loading, setLoading] = useState(true); const getAgents = async () => { @@ -43,6 +52,7 @@ function AgentsList() { if (!response.ok) throw new Error('Failed to fetch agents'); const data = await response.json(); setUserAgents(data); + dispatch(setAgents(data)); setLoading(false); } catch (error) { console.error('Error:', error); @@ -52,6 +62,7 @@ function AgentsList() { useEffect(() => { getAgents(); + if (selectedAgent) dispatch(setSelectedAgent(null)); }, [token]); return (
@@ -62,6 +73,7 @@ function AgentsList() { Discover and create custom versions of DocsGPT that combine instructions, extra knowledge, and any combination of skills.

+ {/* Premade agents section */} {/*

Premade by DocsGPT @@ -126,6 +138,7 @@ function AgentsList() { )) @@ -148,9 +161,11 @@ function AgentsList() { function AgentCard({ agent, + agents, setUserAgents, }: { agent: Agent; + agents: Agent[]; setUserAgents: React.Dispatch>; }) { const navigate = useNavigate(); @@ -200,8 +215,10 @@ function AgentCard({ ]; const handleClick = () => { - dispatch(setSelectedAgent(agent)); - navigate(`/`); + if (agent.status === 'published') { + dispatch(setSelectedAgent(agent)); + navigate(`/`); + } }; const handleDelete = async (agentId: string) => { @@ -211,11 +228,15 @@ function AgentCard({ setUserAgents((prevAgents) => prevAgents.filter((prevAgent) => prevAgent.id !== data.id), ); + dispatch(setAgents(agents.filter((prevAgent) => prevAgent.id !== data.id))); }; return (
handleClick()} + className={`relative flex h-44 w-48 flex-col justify-between rounded-[1.2rem] bg-[#F6F6F6] px-6 py-5 hover:bg-[#ECECEC] dark:bg-[#383838] hover:dark:bg-[#383838]/80 ${agent.status === 'published' && 'cursor-pointer'}`} + onClick={(e) => { + e.stopPropagation(); + handleClick(); + }} >
`/api/update_agent/${agent_id}`, DELETE_AGENT: (id: string) => `/api/delete_agent?id=${id}`, + AGENT_WEBHOOK: (id: string) => `/api/agent_webhook?id=${id}`, PROMPTS: '/api/get_prompts', CREATE_PROMPT: '/api/create_prompt', DELETE_PROMPT: '/api/delete_prompt', diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index bbe20b10..4a0f45d8 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -31,6 +31,8 @@ const userService = { apiClient.put(endpoints.USER.UPDATE_AGENT(agent_id), data, token), deleteAgent: (id: string, token: string | null): Promise => apiClient.delete(endpoints.USER.DELETE_AGENT(id), token), + getAgentWebhook: (id: string, token: string | null): Promise => + apiClient.get(endpoints.USER.AGENT_WEBHOOK(id), token), getPrompts: (token: string | null): Promise => apiClient.get(endpoints.USER.PROMPTS, token), createPrompt: (data: any, token: string | null): Promise => diff --git a/frontend/src/assets/monitoring-purple.svg b/frontend/src/assets/monitoring-purple.svg new file mode 100644 index 00000000..ef849521 --- /dev/null +++ b/frontend/src/assets/monitoring-purple.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/assets/monitoring-white.svg b/frontend/src/assets/monitoring-white.svg new file mode 100644 index 00000000..b015eeee --- /dev/null +++ b/frontend/src/assets/monitoring-white.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/components/CopyButton.tsx b/frontend/src/components/CopyButton.tsx index c430603f..0afbbe82 100644 --- a/frontend/src/components/CopyButton.tsx +++ b/frontend/src/components/CopyButton.tsx @@ -1,58 +1,136 @@ +import clsx from 'clsx'; import copy from 'copy-to-clipboard'; -import { useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import CheckMark from '../assets/checkmark.svg?react'; -import Copy from '../assets/copy.svg?react'; +import CopyIcon from '../assets/copy.svg?react'; + +type CopyButtonProps = { + textToCopy: string; + bgColorLight?: string; + bgColorDark?: string; + hoverBgColorLight?: string; + hoverBgColorDark?: string; + iconSize?: string; + padding?: string; + showText?: boolean; + copiedDuration?: number; + className?: string; + iconWrapperClassName?: string; + textClassName?: string; +}; + +const DEFAULT_ICON_SIZE = 'w-4 h-4'; +const DEFAULT_PADDING = 'p-2'; +const DEFAULT_COPIED_DURATION = 2000; +const DEFAULT_BG_LIGHT = '#FFFFFF'; +const DEFAULT_BG_DARK = 'transparent'; +const DEFAULT_HOVER_BG_LIGHT = '#EEEEEE'; +const DEFAULT_HOVER_BG_DARK = '#4A4A4A'; export default function CopyButton({ - text, - colorLight, - colorDark, + textToCopy, + bgColorLight = DEFAULT_BG_LIGHT, + bgColorDark = DEFAULT_BG_DARK, + hoverBgColorLight = DEFAULT_HOVER_BG_LIGHT, + hoverBgColorDark = DEFAULT_HOVER_BG_DARK, + iconSize = DEFAULT_ICON_SIZE, + padding = DEFAULT_PADDING, showText = false, -}: { - text: string; - colorLight?: string; - colorDark?: string; - showText?: boolean; -}) { + copiedDuration = DEFAULT_COPIED_DURATION, + className, + iconWrapperClassName, + textClassName, +}: CopyButtonProps) { const { t } = useTranslation(); - const [copied, setCopied] = useState(false); - const [isCopyHovered, setIsCopyHovered] = useState(false); + const [isCopied, setIsCopied] = useState(false); + const timeoutIdRef = useRef(null); - const handleCopyClick = (text: string) => { - copy(text); - setCopied(true); - setTimeout(() => { - setCopied(false); - }, 3000); - }; + const iconWrapperClasses = clsx( + 'flex items-center justify-center rounded-full transition-colors duration-150 ease-in-out', + padding, + `bg-[${bgColorLight}] dark:bg-[${bgColorDark}]`, + `hover:bg-[${hoverBgColorLight}] dark:hover:bg-[${hoverBgColorDark}]`, + { + 'bg-green-100 dark:bg-green-900 hover:bg-green-100 dark:hover:bg-green-900': + isCopied, + }, + iconWrapperClassName, + ); + const rootButtonClasses = clsx( + 'flex items-center gap-2 group', + 'focus:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-blue-500 rounded-full', + className, + ); + + const textSpanClasses = clsx( + 'text-xs text-gray-600 dark:text-gray-400 transition-opacity duration-150 ease-in-out', + { 'opacity-75': isCopied }, + textClassName, + ); + + const IconComponent = isCopied ? CheckMark : CopyIcon; + const iconClasses = clsx(iconSize, { + 'stroke-green-600 dark:stroke-green-400': isCopied, + 'fill-none text-gray-700 dark:text-gray-300': !isCopied, + }); + + const buttonTitle = isCopied + ? t('conversation.copied') + : t('conversation.copy'); + const displayedText = isCopied + ? t('conversation.copied') + : t('conversation.copy'); + + const handleCopy = useCallback(() => { + if (isCopied) return; + + try { + const success = copy(textToCopy); + if (success) { + setIsCopied(true); + + if (timeoutIdRef.current) { + clearTimeout(timeoutIdRef.current); + } + + timeoutIdRef.current = setTimeout(() => { + setIsCopied(false); + timeoutIdRef.current = null; + }, copiedDuration); + } else { + console.warn('Copy command failed.'); + } + } catch (error) { + console.error('Failed to copy text:', error); + } + }, [textToCopy, copiedDuration, isCopied]); + + useEffect(() => { + return () => { + if (timeoutIdRef.current) { + clearTimeout(timeoutIdRef.current); + } + }; + }, []); return ( ); } diff --git a/frontend/src/components/MessageInput.tsx b/frontend/src/components/MessageInput.tsx index e7ef7f9d..60cd4b81 100644 --- a/frontend/src/components/MessageInput.tsx +++ b/frontend/src/components/MessageInput.tsx @@ -36,15 +36,7 @@ type MessageInputProps = { loading: boolean; showSourceButton?: boolean; showToolButton?: boolean; -}; - -type UploadState = { - taskId: string; - fileName: string; - progress: number; - attachment_id?: string; - token_count?: number; - status: 'uploading' | 'processing' | 'completed' | 'failed'; + autoFocus?: boolean; }; export default function MessageInput({ @@ -54,6 +46,7 @@ export default function MessageInput({ loading, showSourceButton = true, showToolButton = true, + autoFocus = true, }: MessageInputProps) { const { t } = useTranslation(); const [isDarkTheme] = useDarkTheme(); @@ -235,7 +228,7 @@ export default function MessageInput({ }; useEffect(() => { - inputRef.current?.focus(); + if (autoFocus) inputRef.current?.focus(); handleInput(); }, []); diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx index a241b2d3..a7c8467d 100644 --- a/frontend/src/conversation/ConversationBubble.tsx +++ b/frontend/src/conversation/ConversationBubble.tsx @@ -5,10 +5,7 @@ import { useTranslation } from 'react-i18next'; import ReactMarkdown from 'react-markdown'; import { useSelector } from 'react-redux'; import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'; -import { - oneLight, - vscDarkPlus, -} from 'react-syntax-highlighter/dist/cjs/styles/prism'; +import { oneLight, vscDarkPlus } from 'react-syntax-highlighter/dist/cjs/styles/prism'; import rehypeKatex from 'rehype-katex'; import remarkGfm from 'remark-gfm'; import remarkMath from 'remark-math'; @@ -29,10 +26,7 @@ import CopyButton from '../components/CopyButton'; import Sidebar from '../components/Sidebar'; import SpeakButton from '../components/TextToSpeechButton'; import { useDarkTheme, useOutsideAlerter } from '../hooks'; -import { - selectChunks, - selectSelectedDocs, -} from '../preferences/preferenceSlice'; +import { selectChunks, selectSelectedDocs } from '../preferences/preferenceSlice'; import classes from './ConversationBubble.module.css'; import { FEEDBACK, MESSAGE_TYPE } from './conversationModels'; import { ToolCallsType } from './types'; @@ -377,7 +371,7 @@ const ConversationBubble = forwardRef< {language}
- +
{' '}

@@ -689,7 +683,7 @@ function ToolCalls({ toolCalls }: { toolCalls: ToolCallsType[] }) { Response {' '}

@@ -766,7 +760,7 @@ function Thought({ {language}

(null); + const [apiKey, setApiKey] = useState(null); + const [webhookUrl, setWebhookUrl] = useState(null); + const [loadingStates, setLoadingStates] = useState({ + publicLink: false, + apiKey: false, + webhook: false, + }); + + const setLoading = ( + key: 'publicLink' | 'apiKey' | 'webhook', + state: boolean, + ) => { + setLoadingStates((prev) => ({ ...prev, [key]: state })); + }; + + const handleGenerateWebhook = async () => { + setLoading('webhook', true); + const response = await userService.getAgentWebhook(agent.id ?? '', token); + if (!response.ok) { + setLoading('webhook', false); + return; + } + const data = await response.json(); + setWebhookUrl(data.webhook_url); + setLoading('webhook', false); + }; + if (modalState !== 'ACTIVE') return null; return ( { - // if (mode === 'new') navigate('/agents'); setModalState('INACTIVE'); }} > @@ -54,12 +88,34 @@ export default function AgentDetailsModal({ )}

-

- Webhooks -

- +
+

+ Webhook URL +

+ {webhookUrl && ( +
+ +
+ )} +
+ {webhookUrl ? ( +
+

+ {webhookUrl} +

+
+ ) : ( + + )}
diff --git a/frontend/src/modals/ConfirmationModal.tsx b/frontend/src/modals/ConfirmationModal.tsx index 25f8c2da..28151736 100644 --- a/frontend/src/modals/ConfirmationModal.tsx +++ b/frontend/src/modals/ConfirmationModal.tsx @@ -40,19 +40,23 @@ export default function ConfirmationModal({ >
-

+

{message}