diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 74f8a4a9..31aa43ac 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -3,7 +3,6 @@ import datetime import json import logging import os -import sys import traceback from bson.dbref import DBRef @@ -263,13 +262,12 @@ def complete_stream( data = json.dumps({"type": "end"}) yield f"data: {data}\n\n" except Exception as e: - print("\033[91merr", str(e), file=sys.stderr) - traceback.print_exc() + logger.error(f"Error in stream: {str(e)}") + logger.error(traceback.format_exc()) data = json.dumps( { "type": "error", "error": "Please try again later. We apologize for any inconvenience.", - "error_exception": str(e), } ) yield f"data: {data}\n\n" @@ -384,7 +382,7 @@ class Stream(Resource): except ValueError: message = "Malformed request body" - print("\033[91merr", str(message), file=sys.stderr) + current_app.logger.error(f"/stream - error: {message}") return Response( error_stream_generate(message), status=400, @@ -395,13 +393,9 @@ class Stream(Resource): f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}", extra={"error": str(e), "traceback": traceback.format_exc()}, ) - message = e.args[0] status_code = 400 - # Custom exceptions with two arguments, index 1 as status code - if len(e.args) >= 2: - status_code = e.args[1] return Response( - error_stream_generate(message), + error_stream_generate('Unknown error occurred'), status=status_code, mimetype="text/event-stream", ) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 6c18a680..4efc4cb7 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -8,7 +8,7 @@ import json from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef from bson.objectid import ObjectId -from flask import Blueprint, jsonify, make_response, redirect, request +from flask import Blueprint, current_app, jsonify, make_response, redirect, request from flask_restx import fields, inputs, Namespace, Resource from werkzeug.utils import secure_filename @@ -83,7 +83,8 @@ class DeleteConversation(Resource): try: conversations_collection.delete_one({"_id": ObjectId(conversation_id)}) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting conversation: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -97,7 +98,8 @@ class DeleteAllConversations(Resource): try: conversations_collection.delete_many({"user": user_id}) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting all conversations: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -114,7 +116,8 @@ class GetConversations(Resource): for conversation in conversations ] except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving conversations: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(list_conversations), 200) @@ -138,7 +141,8 @@ class GetSingleConversation(Resource): if not conversation: return make_response(jsonify({"status": "not found"}), 404) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving conversation: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(conversation["queries"]), 200) @@ -170,7 +174,8 @@ class UpdateConversationName(Resource): {"_id": ObjectId(data["id"])}, {"$set": {"name": data["name"]}} ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating conversation name: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -221,7 +226,8 @@ class SubmitFeedback(Resource): ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error submitting feedback: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -244,7 +250,8 @@ class DeleteByIds(Resource): if result: return make_response(jsonify({"success": True}), 200) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting indexes: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": False}), 400) @@ -277,7 +284,8 @@ class DeleteOldIndexes(Resource): except FileNotFoundError: pass except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting old indexes: {err}") + return make_response(jsonify({"success": False}), 400) sources_collection.delete_one({"_id": ObjectId(source_id)}) return make_response(jsonify({"success": True}), 200) @@ -389,8 +397,8 @@ class UploadFile(Resource): ) except Exception as err: - print(f"Error: {err}") - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error uploading file: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True, "task_id": task.id}), 200) @@ -439,7 +447,8 @@ class UploadRemote(Resource): loader=data["source"] ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error uploading remote source: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True, "task_id": task.id}), 200) @@ -471,7 +480,8 @@ class TaskStatus(Resource): ): task_meta = str(task_meta) # Convert to a string representation except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting task status: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"status": task.status, "result": task_meta}), 200) @@ -546,7 +556,8 @@ class PaginatedSources(Resource): return make_response(jsonify(response), 200) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving paginated sources: {err}") + return make_response(jsonify({"success": False}), 400) @user_ns.route("/api/sources") @@ -606,7 +617,8 @@ class CombinedJson(Resource): ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving sources: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(data), 200) @@ -632,7 +644,8 @@ class CheckDocs(Resource): if os.path.exists(vectorstore) or data["docs"] == "default": return {"status": "exists"}, 200 except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error checking document: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"status": "not found"}), 404) @@ -670,7 +683,8 @@ class CreatePrompt(Resource): ) new_id = str(resp.inserted_id) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error creating prompt: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"id": new_id}), 200) @@ -697,7 +711,8 @@ class GetPrompts(Resource): } ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving prompts: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(list_prompts), 200) @@ -738,7 +753,8 @@ class GetSinglePrompt(Resource): prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)}) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving prompt: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"content": prompt["content"]}), 200) @@ -762,7 +778,8 @@ class DeletePrompt(Resource): try: prompts_collection.delete_one({"_id": ObjectId(data["id"])}) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error deleting prompt: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -795,7 +812,8 @@ class UpdatePrompt(Resource): {"$set": {"name": data["name"], "content": data["content"]}}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating prompt: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -830,7 +848,8 @@ class GetApiKeys(Resource): } ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error retrieving API keys: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify(list_keys), 200) @@ -874,7 +893,8 @@ class CreateApiKey(Resource): resp = api_key_collection.insert_one(new_api_key) new_id = str(resp.inserted_id) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error creating API key: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"id": new_id, "key": key}), 201) @@ -900,7 +920,8 @@ class DeleteApiKey(Resource): if result.deleted_count == 0: return {"success": False, "message": "API Key not found"}, 404 except Exception as err: - return {"success": False, "error": str(err)}, 400 + current_app.logger.error(f"Error deleting API key: {err}") + return {"success": False}, 400 return {"success": True}, 200 @@ -1100,7 +1121,8 @@ class ShareConversation(Resource): 201, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error sharing conversation: {err}") + return make_response(jsonify({"success": False}), 400) @user_ns.route("/api/shared_conversation/") @@ -1155,7 +1177,8 @@ class GetPubliclySharedConversations(Resource): res["api_key"] = shared["api_key"] return make_response(jsonify(res), 200) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting shared conversation: {err}") + return make_response(jsonify({"success": False}), 400) @user_ns.route("/api/get_message_analytics") @@ -1196,7 +1219,8 @@ class GetMessageAnalytics(Resource): else None ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting API key: {err}") + return make_response(jsonify({"success": False}), 400) end_date = datetime.datetime.now(datetime.timezone.utc) if filter_option == "last_hour": @@ -1289,7 +1313,8 @@ class GetMessageAnalytics(Resource): daily_messages[entry["_id"]["day"]] = entry["total_messages"] except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting message analytics: {err}") + return make_response(jsonify({"success": False}), 400) return make_response( jsonify({"success": True, "messages": daily_messages}), 200 @@ -1331,7 +1356,8 @@ class GetTokenAnalytics(Resource): else None ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting API key: {err}") + return make_response(jsonify({"success": False}), 400) end_date = datetime.datetime.now(datetime.timezone.utc) if filter_option == "last_hour": @@ -1440,7 +1466,8 @@ class GetTokenAnalytics(Resource): daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"] except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting token analytics: {err}") + return make_response(jsonify({"success": False}), 400) return make_response( jsonify({"success": True, "token_usage": daily_token_usage}), 200 @@ -1482,7 +1509,8 @@ class GetFeedbackAnalytics(Resource): else None ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting API key: {err}") + return make_response(jsonify({"success": False}), 400) end_date = datetime.datetime.now(datetime.timezone.utc) @@ -1583,7 +1611,8 @@ class GetFeedbackAnalytics(Resource): } except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting feedback analytics: {err}") + return make_response(jsonify({"success": False}), 400) return make_response( jsonify({"success": True, "feedback": daily_feedback}), 200 @@ -1625,7 +1654,8 @@ class GetUserLogs(Resource): else None ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting API key: {err}") + return make_response(jsonify({"success": False}), 400) query = {} if api_key: @@ -1709,7 +1739,8 @@ class ManageSync(Resource): update_data, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating sync frequency: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -1744,7 +1775,8 @@ class TextToSpeech(Resource): 200, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error synthesizing audio: {err}") + return make_response(jsonify({"success": False}), 400) @user_ns.route("/api/available_tools") @@ -1768,7 +1800,8 @@ class AvailableTools(Resource): } ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting available tools: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True, "data": tools_metadata}), 200) @@ -1786,7 +1819,8 @@ class GetTools(Resource): tool.pop("_id") user_tools.append(tool) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error getting user tools: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True, "tools": user_tools}), 200) @@ -1858,7 +1892,8 @@ class CreateTool(Resource): resp = user_tools_collection.insert_one(new_tool) new_id = str(resp.inserted_id) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error creating tool: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"id": new_id}), 200) @@ -1909,7 +1944,8 @@ class UpdateTool(Resource): {"$set": update_data}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating tool: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -1941,7 +1977,8 @@ class UpdateToolConfig(Resource): {"$set": {"config": data["config"]}}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating tool config: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -1975,7 +2012,8 @@ class UpdateToolActions(Resource): {"$set": {"actions": data["actions"]}}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating tool actions: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -2007,7 +2045,8 @@ class UpdateToolStatus(Resource): {"$set": {"status": data["status"]}}, ) except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + current_app.logger.error(f"Error updating tool status: {err}") + return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) @@ -2033,6 +2072,7 @@ class DeleteTool(Resource): if result.deleted_count == 0: return {"success": False, "message": "Tool not found"}, 404 except Exception as err: - return {"success": False, "error": str(err)}, 400 + current_app.logger.error(f"Error deleting tool: {err}") + return {"success": False}, 400 return {"success": True}, 200 diff --git a/application/parser/file/docs_parser.py b/application/parser/file/docs_parser.py index 55d45a64..a1295290 100644 --- a/application/parser/file/docs_parser.py +++ b/application/parser/file/docs_parser.py @@ -24,26 +24,27 @@ class PDFParser(BaseParser): # alternatively you can use local vision capable LLM with open(file, "rb") as file_loaded: files = {'file': file_loaded} - response = requests.post(doc2md_service, files=files) - data = response.json()["markdown"] + response = requests.post(doc2md_service, files=files) + data = response.json()["markdown"] return data try: - import PyPDF2 + from pypdf import PdfReader except ImportError: - raise ValueError("PyPDF2 is required to read PDF files.") + raise ValueError("pypdf is required to read PDF files.") text_list = [] with open(file, "rb") as fp: # Create a PDF object - pdf = PyPDF2.PdfReader(fp) + pdf = PdfReader(fp) # Get the number of pages in the PDF document num_pages = len(pdf.pages) # Iterate over every page - for page in range(num_pages): + for page_index in range(num_pages): # Extract the text from the page - page_text = pdf.pages[page].extract_text() + page = pdf.pages[page_index] + page_text = page.extract_text() text_list.append(page_text) text = "\n".join(text_list) @@ -66,4 +67,4 @@ class DocxParser(BaseParser): text = docx2txt.process(file) - return text + return text \ No newline at end of file diff --git a/application/requirements.txt b/application/requirements.txt index 12ea4ee5..5732809b 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -66,7 +66,7 @@ pydantic==2.10.4 pydantic-core==2.27.2 pydantic-settings==2.7.1 pymongo==4.10.1 -pypdf2==3.0.1 +pypdf==5.2.0 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 python-pptx==1.0.2 diff --git a/deployment/docker-compose-azure.yaml b/deployment/docker-compose-azure.yaml index 601831e5..9e8b6fce 100644 --- a/deployment/docker-compose-azure.yaml +++ b/deployment/docker-compose-azure.yaml @@ -1,6 +1,6 @@ services: frontend: - build: ./frontend + build: ../frontend environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING @@ -10,7 +10,7 @@ services: - backend backend: - build: ./application + build: ../application environment: - API_KEY=$OPENAI_API_KEY - EMBEDDINGS_KEY=$OPENAI_API_KEY @@ -25,15 +25,15 @@ services: ports: - "7091:7091" volumes: - - ./application/indexes:/app/application/indexes - - ./application/inputs:/app/application/inputs - - ./application/vectors:/app/application/vectors + - ../application/indexes:/app/application/indexes + - ../application/inputs:/app/application/inputs + - ../application/vectors:/app/application/vectors depends_on: - redis - mongo worker: - build: ./application + build: ../application command: celery -A application.app.celery worker -l INFO environment: - API_KEY=$OPENAI_API_KEY diff --git a/deployment/docker-compose-local.yaml b/deployment/docker-compose-local.yaml index d9fd248b..77a82866 100644 --- a/deployment/docker-compose-local.yaml +++ b/deployment/docker-compose-local.yaml @@ -1,8 +1,8 @@ services: frontend: - build: ./frontend + build: ../frontend volumes: - - ./frontend/src:/app/src + - ../frontend/src:/app/src environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index b1d083a0..15d9522f 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -1,8 +1,8 @@ services: frontend: - build: ./frontend + build: ../frontend volumes: - - ./frontend/src:/app/src + - ../frontend/src:/app/src environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING @@ -12,7 +12,7 @@ services: - backend backend: - build: ./application + build: ../application environment: - API_KEY=$API_KEY - EMBEDDINGS_KEY=$API_KEY @@ -26,15 +26,15 @@ services: ports: - "7091:7091" volumes: - - ./application/indexes:/app/application/indexes - - ./application/inputs:/app/application/inputs - - ./application/vectors:/app/application/vectors + - ../application/indexes:/app/application/indexes + - ../application/inputs:/app/application/inputs + - ../application/vectors:/app/application/vectors depends_on: - redis - mongo worker: - build: ./application + build: ../application command: celery -A application.app.celery worker -l INFO -B environment: - API_KEY=$API_KEY