LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 08:33:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/edit-chunks

Browse Source
This commit is contained in:
Siddhant Rai
2025-02-08 14:56:34 +05:30
committed by GitHub
parent 0fc9718c35 5d17072709
commit 82b2be5046
58 changed files with 258 additions and 2008 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
application/Dockerfile
View File

@@ -6,7 +6,6 @@ ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
# Install necessary packages and Python
apt-get update && \
apt-get install -y --no-install-recommends gcc wget unzip libc6-dev python3.12 python3.12-venv && \
rm -rf /var/lib/apt/lists/*
@@ -49,7 +48,6 @@ FROM ubuntu:24.04 as final
RUN apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
# Install Python
apt-get update && apt-get install -y --no-install-recommends python3.12 && \
ln -s /usr/bin/python3.12 /usr/bin/python && \
rm -rf /var/lib/apt/lists/*

14
application/api/answer/routes.py
View File

@@ -3,7 +3,6 @@ import datetime
import json
import logging
import os
import sys
import traceback
from bson.dbref import DBRef
@@ -263,13 +262,12 @@ def complete_stream(
data = json.dumps({"type": "end"})
yield f"data: {data}\n\n"
except Exception as e:
print("\033[91merr", str(e), file=sys.stderr)
traceback.print_exc()
logger.error(f"Error in stream: {str(e)}")
logger.error(traceback.format_exc())
data = json.dumps(
{
"type": "error",
"error": "Please try again later. We apologize for any inconvenience.",
"error_exception": str(e),
}
)
yield f"data: {data}\n\n"
@@ -384,7 +382,7 @@ class Stream(Resource):
except ValueError:
message = "Malformed request body"
print("\033[91merr", str(message), file=sys.stderr)
current_app.logger.error(f"/stream - error: {message}")
return Response(
error_stream_generate(message),
status=400,
@@ -395,13 +393,9 @@ class Stream(Resource):
f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
extra={"error": str(e), "traceback": traceback.format_exc()},
)
message = e.args[0]
status_code = 400
# Custom exceptions with two arguments, index 1 as status code
if len(e.args) >= 2:
status_code = e.args[1]
return Response(
error_stream_generate(message),
error_stream_generate('Unknown error occurred'),
status=status_code,
mimetype="text/event-stream",
)

128
application/api/user/routes.py
View File

@@ -7,7 +7,7 @@ import uuid
from bson.binary import Binary, UuidRepresentation
from bson.dbref import DBRef
from bson.objectid import ObjectId
from flask import Blueprint, jsonify, make_response, redirect, request
from flask import Blueprint, current_app, jsonify, make_response, redirect, request
from flask_restx import fields, inputs, Namespace, Resource
from werkzeug.utils import secure_filename
@@ -82,7 +82,8 @@ class DeleteConversation(Resource):
try:
conversations_collection.delete_one({"_id": ObjectId(conversation_id)})
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting conversation: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -96,7 +97,8 @@ class DeleteAllConversations(Resource):
try:
conversations_collection.delete_many({"user": user_id})
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting all conversations: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -113,7 +115,8 @@ class GetConversations(Resource):
for conversation in conversations
]
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving conversations: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(list_conversations), 200)
@@ -137,7 +140,8 @@ class GetSingleConversation(Resource):
if not conversation:
return make_response(jsonify({"status": "not found"}), 404)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving conversation: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(conversation["queries"]), 200)
@@ -169,7 +173,8 @@ class UpdateConversationName(Resource):
{"_id": ObjectId(data["id"])}, {"$set": {"name": data["name"]}}
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating conversation name: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -220,7 +225,8 @@ class SubmitFeedback(Resource):
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error submitting feedback: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -243,7 +249,8 @@ class DeleteByIds(Resource):
if result:
return make_response(jsonify({"success": True}), 200)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting indexes: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": False}), 400)
@@ -276,7 +283,8 @@ class DeleteOldIndexes(Resource):
except FileNotFoundError:
pass
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting old indexes: {err}")
return make_response(jsonify({"success": False}), 400)
sources_collection.delete_one({"_id": ObjectId(source_id)})
return make_response(jsonify({"success": True}), 200)
@@ -388,8 +396,8 @@ class UploadFile(Resource):
)
except Exception as err:
print(f"Error: {err}")
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error uploading file: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True, "task_id": task.id}), 200)
@@ -434,7 +442,8 @@ class UploadRemote(Resource):
loader=loader,
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error uploading remote source: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True, "task_id": task.id}), 200)
@@ -466,7 +475,8 @@ class TaskStatus(Resource):
):
task_meta = str(task_meta) # Convert to a string representation
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting task status: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"status": task.status, "result": task_meta}), 200)
@@ -541,7 +551,8 @@ class PaginatedSources(Resource):
return make_response(jsonify(response), 200)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving paginated sources: {err}")
return make_response(jsonify({"success": False}), 400)
@user_ns.route("/api/sources")
@@ -601,7 +612,8 @@ class CombinedJson(Resource):
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving sources: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(data), 200)
@@ -627,7 +639,8 @@ class CheckDocs(Resource):
if os.path.exists(vectorstore) or data["docs"] == "default":
return {"status": "exists"}, 200
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error checking document: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"status": "not found"}), 404)
@@ -665,7 +678,8 @@ class CreatePrompt(Resource):
)
new_id = str(resp.inserted_id)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error creating prompt: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"id": new_id}), 200)
@@ -692,7 +706,8 @@ class GetPrompts(Resource):
}
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving prompts: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(list_prompts), 200)
@@ -733,7 +748,8 @@ class GetSinglePrompt(Resource):
prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving prompt: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"content": prompt["content"]}), 200)
@@ -757,7 +773,8 @@ class DeletePrompt(Resource):
try:
prompts_collection.delete_one({"_id": ObjectId(data["id"])})
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error deleting prompt: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -790,7 +807,8 @@ class UpdatePrompt(Resource):
{"$set": {"name": data["name"], "content": data["content"]}},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating prompt: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -825,7 +843,8 @@ class GetApiKeys(Resource):
}
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error retrieving API keys: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify(list_keys), 200)
@@ -869,7 +888,8 @@ class CreateApiKey(Resource):
resp = api_key_collection.insert_one(new_api_key)
new_id = str(resp.inserted_id)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error creating API key: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"id": new_id, "key": key}), 201)
@@ -895,7 +915,8 @@ class DeleteApiKey(Resource):
if result.deleted_count == 0:
return {"success": False, "message": "API Key not found"}, 404
except Exception as err:
return {"success": False, "error": str(err)}, 400
current_app.logger.error(f"Error deleting API key: {err}")
return {"success": False}, 400
return {"success": True}, 200
@@ -1095,7 +1116,8 @@ class ShareConversation(Resource):
201,
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error sharing conversation: {err}")
return make_response(jsonify({"success": False}), 400)
@user_ns.route("/api/shared_conversation/<string:identifier>")
@@ -1150,7 +1172,8 @@ class GetPubliclySharedConversations(Resource):
res["api_key"] = shared["api_key"]
return make_response(jsonify(res), 200)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting shared conversation: {err}")
return make_response(jsonify({"success": False}), 400)
@user_ns.route("/api/get_message_analytics")
@@ -1191,7 +1214,8 @@ class GetMessageAnalytics(Resource):
else None
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting API key: {err}")
return make_response(jsonify({"success": False}), 400)
end_date = datetime.datetime.now(datetime.timezone.utc)
if filter_option == "last_hour":
@@ -1284,7 +1308,8 @@ class GetMessageAnalytics(Resource):
daily_messages[entry["_id"]["day"]] = entry["total_messages"]
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting message analytics: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(
jsonify({"success": True, "messages": daily_messages}), 200
@@ -1326,7 +1351,8 @@ class GetTokenAnalytics(Resource):
else None
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting API key: {err}")
return make_response(jsonify({"success": False}), 400)
end_date = datetime.datetime.now(datetime.timezone.utc)
if filter_option == "last_hour":
@@ -1435,7 +1461,8 @@ class GetTokenAnalytics(Resource):
daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"]
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting token analytics: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(
jsonify({"success": True, "token_usage": daily_token_usage}), 200
@@ -1477,8 +1504,9 @@ class GetFeedbackAnalytics(Resource):
else None
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting API key: {err}")
return make_response(jsonify({"success": False}), 400)
end_date = datetime.datetime.now(datetime.timezone.utc)
if filter_option == "last_hour":
@@ -1575,7 +1603,8 @@ class GetFeedbackAnalytics(Resource):
}
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting feedback analytics: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(
jsonify({"success": True, "feedback": daily_feedback}), 200
@@ -1617,7 +1646,8 @@ class GetUserLogs(Resource):
else None
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting API key: {err}")
return make_response(jsonify({"success": False}), 400)
query = {}
if api_key:
@@ -1701,7 +1731,8 @@ class ManageSync(Resource):
update_data,
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating sync frequency: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -1736,7 +1767,8 @@ class TextToSpeech(Resource):
200,
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error synthesizing audio: {err}")
return make_response(jsonify({"success": False}), 400)
@user_ns.route("/api/available_tools")
@@ -1760,7 +1792,8 @@ class AvailableTools(Resource):
}
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting available tools: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True, "data": tools_metadata}), 200)
@@ -1778,7 +1811,8 @@ class GetTools(Resource):
tool.pop("_id")
user_tools.append(tool)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error getting user tools: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True, "tools": user_tools}), 200)
@@ -1850,7 +1884,8 @@ class CreateTool(Resource):
resp = user_tools_collection.insert_one(new_tool)
new_id = str(resp.inserted_id)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error creating tool: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"id": new_id}), 200)
@@ -1901,7 +1936,8 @@ class UpdateTool(Resource):
{"$set": update_data},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating tool: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -1933,7 +1969,8 @@ class UpdateToolConfig(Resource):
{"$set": {"config": data["config"]}},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating tool config: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -1967,7 +2004,8 @@ class UpdateToolActions(Resource):
{"$set": {"actions": data["actions"]}},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating tool actions: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -1999,7 +2037,8 @@ class UpdateToolStatus(Resource):
{"$set": {"status": data["status"]}},
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
current_app.logger.error(f"Error updating tool status: {err}")
return make_response(jsonify({"success": False}), 400)
return make_response(jsonify({"success": True}), 200)
@@ -2025,7 +2064,8 @@ class DeleteTool(Resource):
if result.deleted_count == 0:
return {"success": False, "message": "Tool not found"}, 404
except Exception as err:
return {"success": False, "error": str(err)}, 400
current_app.logger.error(f"Error deleting tool: {err}")
return {"success": False}, 400
return {"success": True}, 200

17
application/parser/file/docs_parser.py
View File

@@ -24,26 +24,27 @@ class PDFParser(BaseParser):
# alternatively you can use local vision capable LLM
with open(file, "rb") as file_loaded:
files = {'file': file_loaded}
response = requests.post(doc2md_service, files=files)
data = response.json()["markdown"]
response = requests.post(doc2md_service, files=files)
data = response.json()["markdown"]
return data
try:
import PyPDF2
from pypdf import PdfReader
except ImportError:
raise ValueError("PyPDF2 is required to read PDF files.")
raise ValueError("pypdf is required to read PDF files.")
text_list = []
with open(file, "rb") as fp:
# Create a PDF object
pdf = PyPDF2.PdfReader(fp)
pdf = PdfReader(fp)
# Get the number of pages in the PDF document
num_pages = len(pdf.pages)
# Iterate over every page
for page in range(num_pages):
for page_index in range(num_pages):
# Extract the text from the page
page_text = pdf.pages[page].extract_text()
page = pdf.pages[page_index]
page_text = page.extract_text()
text_list.append(page_text)
text = "\n".join(text_list)
@@ -66,4 +67,4 @@ class DocxParser(BaseParser):
text = docx2txt.process(file)
return text
return text

2
application/requirements.txt
View File

@@ -66,7 +66,7 @@ pydantic==2.10.4
pydantic-core==2.27.2
pydantic-settings==2.7.1
pymongo==4.10.1
pypdf2==3.0.1
pypdf==5.2.0
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
python-pptx==1.0.2