From ad051ed083dbc5d97601717f838608468f3206ac Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 6 Feb 2025 18:51:17 +0000 Subject: [PATCH 1/4] fix: docker compose files --- deployment/docker-compose-azure.yaml | 12 ++++++------ deployment/docker-compose-local.yaml | 4 ++-- deployment/docker-compose.yaml | 14 +++++++------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/deployment/docker-compose-azure.yaml b/deployment/docker-compose-azure.yaml index 601831e5..9e8b6fce 100644 --- a/deployment/docker-compose-azure.yaml +++ b/deployment/docker-compose-azure.yaml @@ -1,6 +1,6 @@ services: frontend: - build: ./frontend + build: ../frontend environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING @@ -10,7 +10,7 @@ services: - backend backend: - build: ./application + build: ../application environment: - API_KEY=$OPENAI_API_KEY - EMBEDDINGS_KEY=$OPENAI_API_KEY @@ -25,15 +25,15 @@ services: ports: - "7091:7091" volumes: - - ./application/indexes:/app/application/indexes - - ./application/inputs:/app/application/inputs - - ./application/vectors:/app/application/vectors + - ../application/indexes:/app/application/indexes + - ../application/inputs:/app/application/inputs + - ../application/vectors:/app/application/vectors depends_on: - redis - mongo worker: - build: ./application + build: ../application command: celery -A application.app.celery worker -l INFO environment: - API_KEY=$OPENAI_API_KEY diff --git a/deployment/docker-compose-local.yaml b/deployment/docker-compose-local.yaml index d9fd248b..77a82866 100644 --- a/deployment/docker-compose-local.yaml +++ b/deployment/docker-compose-local.yaml @@ -1,8 +1,8 @@ services: frontend: - build: ./frontend + build: ../frontend volumes: - - ./frontend/src:/app/src + - ../frontend/src:/app/src environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index b1d083a0..15d9522f 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -1,8 +1,8 @@ services: frontend: - build: ./frontend + build: ../frontend volumes: - - ./frontend/src:/app/src + - ../frontend/src:/app/src environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING @@ -12,7 +12,7 @@ services: - backend backend: - build: ./application + build: ../application environment: - API_KEY=$API_KEY - EMBEDDINGS_KEY=$API_KEY @@ -26,15 +26,15 @@ services: ports: - "7091:7091" volumes: - - ./application/indexes:/app/application/indexes - - ./application/inputs:/app/application/inputs - - ./application/vectors:/app/application/vectors + - ../application/indexes:/app/application/indexes + - ../application/inputs:/app/application/inputs + - ../application/vectors:/app/application/vectors depends_on: - redis - mongo worker: - build: ./application + build: ../application command: celery -A application.app.celery worker -l INFO -B environment: - API_KEY=$API_KEY From 0c4c4d5622b0f6d2a9c901ad564572fd16f97140 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 6 Feb 2025 19:44:09 +0000 Subject: [PATCH 2/4] fix: improve error logging --- application/api/answer/routes.py | 13 +--- application/api/user/routes.py | 126 ++++++++++++++++++++----------- 2 files changed, 87 insertions(+), 52 deletions(-) diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 74f8a4a9..3044b8cf 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -263,13 +263,12 @@ def complete_stream( data = json.dumps({"type": "end"}) yield f"data: {data}\n\n" except Exception as e: - print("\033[91merr", str(e), file=sys.stderr) - traceback.print_exc() + logger.error(f"Error in stream: {str(e)}") + logger.error(traceback.format_exc()) data = json.dumps( { "type": "error", "error": "Please try again later. We apologize for any inconvenience.", - "error_exception": str(e), } ) yield f"data: {data}\n\n" @@ -384,7 +383,7 @@ class Stream(Resource): except ValueError: message = "Malformed request body" - print("\033[91merr", str(message), file=sys.stderr) + current_app.logger.error(f"/stream - error: {message}") return Response( error_stream_generate(message), status=400, @@ -395,13 +394,9 @@ class Stream(Resource): f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}", extra={"error": str(e), "traceback": traceback.format_exc()}, ) - message = e.args[0] status_code = 400 - # Custom exceptions with two arguments, index 1 as status code - if len(e.args) >= 2: - status_code = e.args[1] return Response( - error_stream_generate(message), + error_stream_generate('Unknown error occurred'), status=status_code, mimetype="text/event-stream", ) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 10b141c0..d00190db 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -7,7 +7,7 @@ import uuid from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef from bson.objectid import ObjectId -from flask import Blueprint, jsonify, make_response, redirect, request +from flask import Blueprint, current_app, jsonify, make_response, redirect, request from flask_restx import fields, inputs, Namespace, Resource from werkzeug.utils import secure_filename @@ -82,7 +82,8 @@ class DeleteConversation(Resource): try: conversations_collection.delete_one({"_id": ObjectId(conversation_id)}) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting conversation: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -96,7 +97,8 @@ class DeleteAllConversations(Resource): try: conversations_collection.delete_many({"user": user_id}) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting all conversations: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -113,7 +115,8 @@ class GetConversations(Resource): for conversation in conversations ] except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving conversations: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(list_conversations), 200) @@ -137,7 +140,8 @@ class GetSingleConversation(Resource): if not conversation: return make_response(jsonify({"status": "not found"}), 404) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving conversation: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(conversation["queries"]), 200) @@ -169,7 +173,8 @@ class UpdateConversationName(Resource): {"_id": ObjectId(data["id"])}, {"$set": {"name": data["name"]}} ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating conversation name: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -220,7 +225,8 @@ class SubmitFeedback(Resource): ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error submitting feedback: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -243,7 +249,8 @@ class DeleteByIds(Resource): if result: return make_response(jsonify({"success": True}), 200) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting indexes: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": False}), 400) @@ -276,7 +283,8 @@ class DeleteOldIndexes(Resource): except FileNotFoundError: pass except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting old indexes: {err}") + return make_response(jsonify({"success": False}), 400) sources_collection.delete_one({"_id": ObjectId(source_id)}) return make_response(jsonify({"success": True}), 200) @@ -388,8 +396,8 @@ class UploadFile(Resource): ) except Exception as err: - print(f"Error: {err}") - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error uploading file: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True, "task_id": task.id}), 200) @@ -434,7 +442,8 @@ class UploadRemote(Resource): loader=loader, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error uploading remote source: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True, "task_id": task.id}), 200) @@ -466,7 +475,8 @@ class TaskStatus(Resource): ): task_meta = str(task_meta) # Convert to a string representation except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting task status: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"status": task.status, "result": task_meta}), 200) @@ -541,7 +551,8 @@ class PaginatedSources(Resource): return make_response(jsonify(response), 200) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving paginated sources: {err}") + return make_response(jsonify({"success": False}), 400) @user_ns.route("/api/sources") @@ -601,7 +612,8 @@ class CombinedJson(Resource): ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving sources: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(data), 200) @@ -627,7 +639,8 @@ class CheckDocs(Resource): if os.path.exists(vectorstore) or data["docs"] == "default": return {"status": "exists"}, 200 except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error checking document: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"status": "not found"}), 404) @@ -665,7 +678,8 @@ class CreatePrompt(Resource): ) new_id = str(resp.inserted_id) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error creating prompt: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"id": new_id}), 200) @@ -692,7 +706,8 @@ class GetPrompts(Resource): } ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving prompts: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(list_prompts), 200) @@ -733,7 +748,8 @@ class GetSinglePrompt(Resource): prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)}) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving prompt: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"content": prompt["content"]}), 200) @@ -757,7 +773,8 @@ class DeletePrompt(Resource): try: prompts_collection.delete_one({"_id": ObjectId(data["id"])}) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting prompt: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -790,7 +807,8 @@ class UpdatePrompt(Resource): {"$set": {"name": data["name"], "content": data["content"]}}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating prompt: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -825,7 +843,8 @@ class GetApiKeys(Resource): } ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving API keys: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(list_keys), 200) @@ -869,7 +888,8 @@ class CreateApiKey(Resource): resp = api_key_collection.insert_one(new_api_key) new_id = str(resp.inserted_id) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error creating API key: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"id": new_id, "key": key}), 201) @@ -895,7 +915,8 @@ class DeleteApiKey(Resource): if result.deleted_count == 0: return {"success": False, "message": "API Key not found"}, 404 except Exception as err: - return {"success": False, "error": str(err)}, 400 + current_app.logger.error(f"Error deleting API key: {err}") + return {"success": False}, 400 return {"success": True}, 200 @@ -1095,7 +1116,8 @@ class ShareConversation(Resource): 201, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error sharing conversation: {err}") + return make_response(jsonify({"success": False}), 400) @user_ns.route("/api/shared_conversation/") @@ -1150,7 +1172,8 @@ class GetPubliclySharedConversations(Resource): res["api_key"] = shared["api_key"] return make_response(jsonify(res), 200) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting shared conversation: {err}") + return make_response(jsonify({"success": False}), 400) @user_ns.route("/api/get_message_analytics") @@ -1191,7 +1214,8 @@ class GetMessageAnalytics(Resource): else None ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting API key: {err}") + return make_response(jsonify({"success": False}), 400) end_date = datetime.datetime.now(datetime.timezone.utc) if filter_option == "last_hour": @@ -1284,7 +1308,8 @@ class GetMessageAnalytics(Resource): daily_messages[entry["_id"]["day"]] = entry["total_messages"] except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting message analytics: {err}") + return make_response(jsonify({"success": False}), 400) return make_response( jsonify({"success": True, "messages": daily_messages}), 200 @@ -1326,7 +1351,8 @@ class GetTokenAnalytics(Resource): else None ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting API key: {err}") + return make_response(jsonify({"success": False}), 400) end_date = datetime.datetime.now(datetime.timezone.utc) if filter_option == "last_hour": @@ -1435,7 +1461,8 @@ class GetTokenAnalytics(Resource): daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"] except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting token analytics: {err}") + return make_response(jsonify({"success": False}), 400) return make_response( jsonify({"success": True, "token_usage": daily_token_usage}), 200 @@ -1477,7 +1504,8 @@ class GetFeedbackAnalytics(Resource): else None ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting API key: {err}") + return make_response(jsonify({"success": False}), 400) end_date = datetime.datetime.now(datetime.timezone.utc) @@ -1578,7 +1606,8 @@ class GetFeedbackAnalytics(Resource): } except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting feedback analytics: {err}") + return make_response(jsonify({"success": False}), 400) return make_response( jsonify({"success": True, "feedback": daily_feedback}), 200 @@ -1620,7 +1649,8 @@ class GetUserLogs(Resource): else None ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting API key: {err}") + return make_response(jsonify({"success": False}), 400) query = {} if api_key: @@ -1704,7 +1734,8 @@ class ManageSync(Resource): update_data, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating sync frequency: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -1739,7 +1770,8 @@ class TextToSpeech(Resource): 200, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error synthesizing audio: {err}") + return make_response(jsonify({"success": False}), 400) @user_ns.route("/api/available_tools") @@ -1763,7 +1795,8 @@ class AvailableTools(Resource): } ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting available tools: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True, "data": tools_metadata}), 200) @@ -1781,7 +1814,8 @@ class GetTools(Resource): tool.pop("_id") user_tools.append(tool) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting user tools: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True, "tools": user_tools}), 200) @@ -1853,7 +1887,8 @@ class CreateTool(Resource): resp = user_tools_collection.insert_one(new_tool) new_id = str(resp.inserted_id) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error creating tool: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"id": new_id}), 200) @@ -1904,7 +1939,8 @@ class UpdateTool(Resource): {"$set": update_data}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating tool: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -1936,7 +1972,8 @@ class UpdateToolConfig(Resource): {"$set": {"config": data["config"]}}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating tool config: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -1970,7 +2007,8 @@ class UpdateToolActions(Resource): {"$set": {"actions": data["actions"]}}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating tool actions: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -2002,7 +2040,8 @@ class UpdateToolStatus(Resource): {"$set": {"status": data["status"]}}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating tool status: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -2028,6 +2067,7 @@ class DeleteTool(Resource): if result.deleted_count == 0: return {"success": False, "message": "Tool not found"}, 404 except Exception as err: - return {"success": False, "error": str(err)}, 400 + current_app.logger.error(f"Error deleting tool: {err}") + return {"success": False}, 400 return {"success": True}, 200 From d819222cf78e855ab16475b7769958002389e472 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 6 Feb 2025 19:46:11 +0000 Subject: [PATCH 3/4] fix: remove unused import --- application/api/answer/routes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 3044b8cf..31aa43ac 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -3,7 +3,6 @@ import datetime import json import logging import os -import sys import traceback from bson.dbref import DBRef From d47232246a9ac49afa8414d4a55519fb85874be6 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 6 Feb 2025 19:59:42 +0000 Subject: [PATCH 4/4] fix: remove old pypdf --- application/parser/file/docs_parser.py | 17 +++++++++-------- application/requirements.txt | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/application/parser/file/docs_parser.py b/application/parser/file/docs_parser.py index 55d45a64..a1295290 100644 --- a/application/parser/file/docs_parser.py +++ b/application/parser/file/docs_parser.py @@ -24,26 +24,27 @@ class PDFParser(BaseParser): # alternatively you can use local vision capable LLM with open(file, "rb") as file_loaded: files = {'file': file_loaded} - response = requests.post(doc2md_service, files=files) - data = response.json()["markdown"] + response = requests.post(doc2md_service, files=files) + data = response.json()["markdown"] return data try: - import PyPDF2 + from pypdf import PdfReader except ImportError: - raise ValueError("PyPDF2 is required to read PDF files.") + raise ValueError("pypdf is required to read PDF files.") text_list = [] with open(file, "rb") as fp: # Create a PDF object - pdf = PyPDF2.PdfReader(fp) + pdf = PdfReader(fp) # Get the number of pages in the PDF document num_pages = len(pdf.pages) # Iterate over every page - for page in range(num_pages): + for page_index in range(num_pages): # Extract the text from the page - page_text = pdf.pages[page].extract_text() + page = pdf.pages[page_index] + page_text = page.extract_text() text_list.append(page_text) text = "\n".join(text_list) @@ -66,4 +67,4 @@ class DocxParser(BaseParser): text = docx2txt.process(file) - return text + return text \ No newline at end of file diff --git a/application/requirements.txt b/application/requirements.txt index 12ea4ee5..5732809b 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -66,7 +66,7 @@ pydantic==2.10.4 pydantic-core==2.27.2 pydantic-settings==2.7.1 pymongo==4.10.1 -pypdf2==3.0.1 +pypdf==5.2.0 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 python-pptx==1.0.2