LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 08:33:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/arc53/DocsGPT

Browse Source
This commit is contained in:
GH Action - Upstream Sync
2025-02-07 01:20:29 +00:00
parent 1aa0af3e58 d47232246a
commit 3e77e79194
7 changed files with 112 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
application/api/answer/routes.py
View File

@@ -3,7 +3,6 @@ import datetime
import json
import logging
import os
import sys
import traceback
from bson.dbref import DBRef
@@ -263,13 +262,12 @@ def complete_stream(
data = json.dumps({"type": "end"})
yield f"data: {data}\n\n"
except Exception as e:
print("\033[91merr", str(e), file=sys.stderr)
traceback.print_exc()
logger.error(f"Error in stream: {str(e)}")
logger.error(traceback.format_exc())
data = json.dumps(
{
"type": "error",
"error": "Please try again later. We apologize for any inconvenience.",
"error_exception": str(e),
}
)
yield f"data: {data}\n\n"
@@ -384,7 +382,7 @@ class Stream(Resource):
except ValueError:
message = "Malformed request body"
print("\033[91merr", str(message), file=sys.stderr)
current_app.logger.error(f"/stream - error: {message}")
return Response(
error_stream_generate(message),
status=400,
@@ -395,13 +393,9 @@ class Stream(Resource):
f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
extra={"error": str(e), "traceback": traceback.format_exc()},
)
message = e.args[0]
status_code = 400
# Custom exceptions with two arguments, index 1 as status code
if len(e.args) >= 2:
status_code = e.args[1]
return Response(
error_stream_generate(message),
error_stream_generate('Unknown error occurred'),
status=status_code,
mimetype="text/event-stream",
)

126
application/api/user/routes.py
View File

@@ -8,7 +8,7 @@ import json
from bson.binary import Binary, UuidRepresentation
from bson.dbref import DBRef
from bson.objectid import ObjectId
from flask import Blueprint, jsonify, make_response, redirect, request
from flask import Blueprint, current_app, jsonify, make_response, redirect, request
from flask_restx import fields, inputs, Namespace, Resource
from werkzeug.utils import secure_filename
@@ -83,7 +83,8 @@ class DeleteConversation(Resource):
try:
conversations_collection.delete_one({"_id": ObjectId(conversation_id)})
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting conversation: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -97,7 +98,8 @@ class DeleteAllConversations(Resource):
try:
conversations_collection.delete_many({"user": user_id})
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting all conversations: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -114,7 +116,8 @@ class GetConversations(Resource):
for conversation in conversations
]
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving conversations: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(list_conversations), 200)
@@ -138,7 +141,8 @@ class GetSingleConversation(Resource):
if not conversation:
return make_response(jsonify({"status": "not found"}), 404)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving conversation: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(conversation["queries"]), 200)
@@ -170,7 +174,8 @@ class UpdateConversationName(Resource):
{"_id": ObjectId(data["id"])}, {"$set": {"name": data["name"]}}
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating conversation name: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -221,7 +226,8 @@ class SubmitFeedback(Resource):
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error submitting feedback: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -244,7 +250,8 @@ class DeleteByIds(Resource):
if result:
return make_response(jsonify({"success": True}), 200)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting indexes: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": False}), 400)
@@ -277,7 +284,8 @@ class DeleteOldIndexes(Resource):
except FileNotFoundError:
pass
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting old indexes: {err}")
return make_response(jsonify({"success": False}), 400)
sources_collection.delete_one({"_id": ObjectId(source_id)})
return make_response(jsonify({"success": True}), 200)
@@ -389,8 +397,8 @@ class UploadFile(Resource):
)
except Exception as err:
print(f"Error: {err}")
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error uploading file: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True, "task_id": task.id}), 200)
@@ -439,7 +447,8 @@ class UploadRemote(Resource):
loader=data["source"]
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error uploading remote source: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True, "task_id": task.id}), 200)
@@ -471,7 +480,8 @@ class TaskStatus(Resource):
):
task_meta = str(task_meta) # Convert to a string representation
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting task status: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"status": task.status, "result": task_meta}), 200)
@@ -546,7 +556,8 @@ class PaginatedSources(Resource):
return make_response(jsonify(response), 200)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving paginated sources: {err}")
return make_response(jsonify({"success": False}), 400)
@user_ns.route("/api/sources")
@@ -606,7 +617,8 @@ class CombinedJson(Resource):
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving sources: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(data), 200)
@@ -632,7 +644,8 @@ class CheckDocs(Resource):
if os.path.exists(vectorstore) or data["docs"] == "default":
return {"status": "exists"}, 200
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error checking document: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"status": "not found"}), 404)
@@ -670,7 +683,8 @@ class CreatePrompt(Resource):
)
new_id = str(resp.inserted_id)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error creating prompt: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"id": new_id}), 200)
@@ -697,7 +711,8 @@ class GetPrompts(Resource):
}
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving prompts: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(list_prompts), 200)
@@ -738,7 +753,8 @@ class GetSinglePrompt(Resource):
prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving prompt: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"content": prompt["content"]}), 200)
@@ -762,7 +778,8 @@ class DeletePrompt(Resource):
try:
prompts_collection.delete_one({"_id": ObjectId(data["id"])})
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting prompt: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -795,7 +812,8 @@ class UpdatePrompt(Resource):
{"$set": {"name": data["name"], "content": data["content"]}},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating prompt: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -830,7 +848,8 @@ class GetApiKeys(Resource):
}
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving API keys: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(list_keys), 200)
@@ -874,7 +893,8 @@ class CreateApiKey(Resource):
resp = api_key_collection.insert_one(new_api_key)
new_id = str(resp.inserted_id)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error creating API key: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"id": new_id, "key": key}), 201)
@@ -900,7 +920,8 @@ class DeleteApiKey(Resource):
if result.deleted_count == 0:
return {"success": False, "message": "API Key not found"}, 404
except Exception as err:
return {"success": False, "error": str(err)}, 400
current_app.logger.error(f"Error deleting API key: {err}")
return {"success": False}, 400
return {"success": True}, 200
@@ -1100,7 +1121,8 @@ class ShareConversation(Resource):
201,
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error sharing conversation: {err}")
return make_response(jsonify({"success": False}), 400)
@user_ns.route("/api/shared_conversation/<string:identifier>")
@@ -1155,7 +1177,8 @@ class GetPubliclySharedConversations(Resource):
res["api_key"] = shared["api_key"]
return make_response(jsonify(res), 200)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting shared conversation: {err}")
return make_response(jsonify({"success": False}), 400)
@user_ns.route("/api/get_message_analytics")
@@ -1196,7 +1219,8 @@ class GetMessageAnalytics(Resource):
else None
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting API key: {err}")
return make_response(jsonify({"success": False}), 400)
end_date = datetime.datetime.now(datetime.timezone.utc)
if filter_option == "last_hour":
@@ -1289,7 +1313,8 @@ class GetMessageAnalytics(Resource):
daily_messages[entry["_id"]["day"]] = entry["total_messages"]
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting message analytics: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(
jsonify({"success": True, "messages": daily_messages}), 200
@@ -1331,7 +1356,8 @@ class GetTokenAnalytics(Resource):
else None
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting API key: {err}")
return make_response(jsonify({"success": False}), 400)
end_date = datetime.datetime.now(datetime.timezone.utc)
if filter_option == "last_hour":
@@ -1440,7 +1466,8 @@ class GetTokenAnalytics(Resource):
daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"]
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting token analytics: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(
jsonify({"success": True, "token_usage": daily_token_usage}), 200
@@ -1482,7 +1509,8 @@ class GetFeedbackAnalytics(Resource):
else None
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting API key: {err}")
return make_response(jsonify({"success": False}), 400)
end_date = datetime.datetime.now(datetime.timezone.utc)
@@ -1583,7 +1611,8 @@ class GetFeedbackAnalytics(Resource):
}
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting feedback analytics: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(
jsonify({"success": True, "feedback": daily_feedback}), 200
@@ -1625,7 +1654,8 @@ class GetUserLogs(Resource):
else None
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting API key: {err}")
return make_response(jsonify({"success": False}), 400)
query = {}
if api_key:
@@ -1709,7 +1739,8 @@ class ManageSync(Resource):
update_data,
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating sync frequency: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -1744,7 +1775,8 @@ class TextToSpeech(Resource):
200,
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error synthesizing audio: {err}")
return make_response(jsonify({"success": False}), 400)
@user_ns.route("/api/available_tools")
@@ -1768,7 +1800,8 @@ class AvailableTools(Resource):
}
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting available tools: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True, "data": tools_metadata}), 200)
@@ -1786,7 +1819,8 @@ class GetTools(Resource):
tool.pop("_id")
user_tools.append(tool)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting user tools: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True, "tools": user_tools}), 200)
@@ -1858,7 +1892,8 @@ class CreateTool(Resource):
resp = user_tools_collection.insert_one(new_tool)
new_id = str(resp.inserted_id)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error creating tool: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"id": new_id}), 200)
@@ -1909,7 +1944,8 @@ class UpdateTool(Resource):
{"$set": update_data},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating tool: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -1941,7 +1977,8 @@ class UpdateToolConfig(Resource):
{"$set": {"config": data["config"]}},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating tool config: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -1975,7 +2012,8 @@ class UpdateToolActions(Resource):
{"$set": {"actions": data["actions"]}},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating tool actions: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -2007,7 +2045,8 @@ class UpdateToolStatus(Resource):
{"$set": {"status": data["status"]}},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating tool status: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -2033,6 +2072,7 @@ class DeleteTool(Resource):
if result.deleted_count == 0:
return {"success": False, "message": "Tool not found"}, 404
except Exception as err:
return {"success": False, "error": str(err)}, 400
current_app.logger.error(f"Error deleting tool: {err}")
return {"success": False}, 400
return {"success": True}, 200

17
application/parser/file/docs_parser.py
View File

@@ -24,26 +24,27 @@ class PDFParser(BaseParser):
# alternatively you can use local vision capable LLM
with open(file, "rb") as file_loaded:
files = {'file': file_loaded}
response = requests.post(doc2md_service, files=files)
data = response.json()["markdown"]
response = requests.post(doc2md_service, files=files)
data = response.json()["markdown"]
return data
try:
import PyPDF2
from pypdf import PdfReader
except ImportError:
raise ValueError("PyPDF2 is required to read PDF files.")
raise ValueError("pypdf is required to read PDF files.")
text_list = []
with open(file, "rb") as fp:
# Create a PDF object
pdf = PyPDF2.PdfReader(fp)
pdf = PdfReader(fp)
# Get the number of pages in the PDF document
num_pages = len(pdf.pages)
# Iterate over every page
for page in range(num_pages):
for page_index in range(num_pages):
# Extract the text from the page
page_text = pdf.pages[page].extract_text()
page = pdf.pages[page_index]
page_text = page.extract_text()
text_list.append(page_text)
text = "\n".join(text_list)
@@ -66,4 +67,4 @@ class DocxParser(BaseParser):
text = docx2txt.process(file)
return text
return text

2
application/requirements.txt
View File

@@ -66,7 +66,7 @@ pydantic==2.10.4
pydantic-core==2.27.2
pydantic-settings==2.7.1
pymongo==4.10.1
pypdf2==3.0.1
pypdf==5.2.0
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
python-pptx==1.0.2