From f0b954dbfbcb9bc3b8c4b878a168bd61913e3aca Mon Sep 17 00:00:00 2001 From: Manish Madan Date: Fri, 10 Oct 2025 20:04:02 +0530 Subject: [PATCH] Upload: communicate failure, minor frontend updates (#2048) * (feat:pause-stream) generator exit * (feat:pause-stream) close request * (feat:pause-stream) finally close; google anthropic * (feat:task_status)communicate failure * (clean:connector) unused routes * (feat:file-table) missing skeletons * (fix:apiKeys) build err --------- Co-authored-by: GH Action - Upstream Sync --- application/api/connector/routes.py | 206 ------------------ application/api/user/sources/upload.py | 11 + .../src/components/ConnectorTreeComponent.tsx | 13 +- frontend/src/components/FileTreeComponent.tsx | 13 +- frontend/src/components/SkeletonLoader.tsx | 12 +- frontend/src/components/UploadToast.tsx | 2 +- frontend/src/locale/en.json | 4 +- frontend/src/locale/es.json | 4 +- frontend/src/locale/jp.json | 4 +- frontend/src/locale/ru.json | 4 +- frontend/src/locale/zh-TW.json | 4 +- frontend/src/locale/zh.json | 4 +- frontend/src/settings/APIKeys.tsx | 2 +- frontend/src/upload/Upload.tsx | 17 +- 14 files changed, 64 insertions(+), 236 deletions(-) diff --git a/application/api/connector/routes.py b/application/api/connector/routes.py index 49307058..d8efc1d3 100644 --- a/application/api/connector/routes.py +++ b/application/api/connector/routes.py @@ -23,15 +23,9 @@ from application.core.settings import settings from application.api import api -from application.utils import ( - check_required_fields -) - - from application.parser.connectors.connector_creator import ConnectorCreator - mongo = MongoDB.get_client() db = mongo[settings.MONGO_DB_NAME] sources_collection = db["sources"] @@ -43,185 +37,6 @@ api.add_namespace(connectors_ns) -@connectors_ns.route("/api/connectors/upload") -class UploadConnector(Resource): - @api.expect( - api.model( - "ConnectorUploadModel", - { - "user": fields.String(required=True, description="User ID"), - "source": fields.String( - required=True, description="Source type (google_drive, github, etc.)" - ), - "name": fields.String(required=True, description="Job name"), - "data": fields.String(required=True, description="Configuration data"), - "repo_url": fields.String(description="GitHub repository URL"), - }, - ) - ) - @api.doc( - description="Uploads connector source for vectorization", - ) - def post(self): - decoded_token = request.decoded_token - if not decoded_token: - return make_response(jsonify({"success": False}), 401) - data = request.form - required_fields = ["user", "source", "name", "data"] - missing_fields = check_required_fields(data, required_fields) - if missing_fields: - return missing_fields - try: - config = json.loads(data["data"]) - source_data = None - sync_frequency = config.get("sync_frequency", "never") - - if data["source"] == "github": - source_data = config.get("repo_url") - elif data["source"] in ["crawler", "url"]: - source_data = config.get("url") - elif data["source"] == "reddit": - source_data = config - elif data["source"] in ConnectorCreator.get_supported_connectors(): - session_token = config.get("session_token") - if not session_token: - return make_response(jsonify({ - "success": False, - "error": f"Missing session_token in {data['source']} configuration" - }), 400) - - file_ids = config.get("file_ids", []) - if isinstance(file_ids, str): - file_ids = [id.strip() for id in file_ids.split(',') if id.strip()] - elif not isinstance(file_ids, list): - file_ids = [] - - folder_ids = config.get("folder_ids", []) - if isinstance(folder_ids, str): - folder_ids = [id.strip() for id in folder_ids.split(',') if id.strip()] - elif not isinstance(folder_ids, list): - folder_ids = [] - - config["file_ids"] = file_ids - config["folder_ids"] = folder_ids - - task = ingest_connector_task.delay( - job_name=data["name"], - user=decoded_token.get("sub"), - source_type=data["source"], - session_token=session_token, - file_ids=file_ids, - folder_ids=folder_ids, - recursive=config.get("recursive", False), - retriever=config.get("retriever", "classic"), - sync_frequency=sync_frequency - ) - return make_response(jsonify({"success": True, "task_id": task.id}), 200) - task = ingest_connector_task.delay( - source_data=source_data, - job_name=data["name"], - user=decoded_token.get("sub"), - loader=data["source"], - sync_frequency=sync_frequency - ) - except Exception as err: - current_app.logger.error( - f"Error uploading connector source: {err}", exc_info=True - ) - return make_response(jsonify({"success": False}), 400) - return make_response(jsonify({"success": True, "task_id": task.id}), 200) - - -@connectors_ns.route("/api/connectors/task_status") -class ConnectorTaskStatus(Resource): - task_status_model = api.model( - "ConnectorTaskStatusModel", - {"task_id": fields.String(required=True, description="Task ID")}, - ) - - @api.expect(task_status_model) - @api.doc(description="Get connector task status") - def get(self): - task_id = request.args.get("task_id") - if not task_id: - return make_response( - jsonify({"success": False, "message": "Task ID is required"}), 400 - ) - try: - from application.celery_init import celery - - task = celery.AsyncResult(task_id) - task_meta = task.info - print(f"Task status: {task.status}") - if not isinstance( - task_meta, (dict, list, str, int, float, bool, type(None)) - ): - task_meta = str(task_meta) - except Exception as err: - current_app.logger.error(f"Error getting task status: {err}", exc_info=True) - return make_response(jsonify({"success": False}), 400) - return make_response(jsonify({"status": task.status, "result": task_meta}), 200) - - -@connectors_ns.route("/api/connectors/sources") -class ConnectorSources(Resource): - @api.doc(description="Get connector sources") - def get(self): - decoded_token = request.decoded_token - if not decoded_token: - return make_response(jsonify({"success": False}), 401) - user = decoded_token.get("sub") - try: - sources = sources_collection.find({"user": user, "type": "connector:file"}).sort("date", -1) - connector_sources = [] - for source in sources: - connector_sources.append({ - "id": str(source["_id"]), - "name": source.get("name"), - "date": source.get("date"), - "type": source.get("type"), - "source": source.get("source"), - "tokens": source.get("tokens", ""), - "retriever": source.get("retriever", "classic"), - "syncFrequency": source.get("sync_frequency", ""), - }) - except Exception as err: - current_app.logger.error(f"Error retrieving connector sources: {err}", exc_info=True) - return make_response(jsonify({"success": False}), 400) - return make_response(jsonify(connector_sources), 200) - - -@connectors_ns.route("/api/connectors/delete") -class DeleteConnectorSource(Resource): - @api.doc( - description="Delete a connector source", - params={"source_id": "The source ID to delete"}, - ) - def delete(self): - decoded_token = request.decoded_token - if not decoded_token: - return make_response(jsonify({"success": False}), 401) - source_id = request.args.get("source_id") - if not source_id: - return make_response( - jsonify({"success": False, "message": "source_id is required"}), 400 - ) - try: - result = sources_collection.delete_one( - {"_id": ObjectId(source_id), "user": decoded_token.get("sub")} - ) - if result.deleted_count == 0: - return make_response( - jsonify({"success": False, "message": "Source not found"}), 404 - ) - except Exception as err: - current_app.logger.error( - f"Error deleting connector source: {err}", exc_info=True - ) - return make_response(jsonify({"success": False}), 400) - return make_response(jsonify({"success": True}), 200) - - @connectors_ns.route("/api/connectors/auth") class ConnectorAuth(Resource): @api.doc(description="Get connector OAuth authorization URL", params={"provider": "Connector provider (e.g., google_drive)"}) @@ -337,27 +152,6 @@ class ConnectorsCallback(Resource): return redirect("/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.") -@connectors_ns.route("/api/connectors/refresh") -class ConnectorRefresh(Resource): - @api.expect(api.model("ConnectorRefreshModel", {"provider": fields.String(required=True), "refresh_token": fields.String(required=True)})) - @api.doc(description="Refresh connector access token") - def post(self): - try: - data = request.get_json() - provider = data.get('provider') - refresh_token = data.get('refresh_token') - - if not provider or not refresh_token: - return make_response(jsonify({"success": False, "error": "provider and refresh_token are required"}), 400) - - auth = ConnectorCreator.create_auth(provider) - token_info = auth.refresh_access_token(refresh_token) - return make_response(jsonify({"success": True, "token_info": token_info}), 200) - except Exception as e: - current_app.logger.error(f"Error refreshing token for connector: {e}") - return make_response(jsonify({"success": False, "error": str(e)}), 500) - - @connectors_ns.route("/api/connectors/files") class ConnectorFiles(Resource): @api.expect(api.model("ConnectorFilesModel", { diff --git a/application/api/user/sources/upload.py b/application/api/user/sources/upload.py index 6c8c692d..7519f7b5 100644 --- a/application/api/user/sources/upload.py +++ b/application/api/user/sources/upload.py @@ -562,10 +562,21 @@ class TaskStatus(Resource): task = celery.AsyncResult(task_id) task_meta = task.info print(f"Task status: {task.status}") + + if task.status == "PENDING": + inspect = celery.control.inspect() + active_workers = inspect.ping() + if not active_workers: + raise ConnectionError("Service unavailable") + if not isinstance( task_meta, (dict, list, str, int, float, bool, type(None)) ): task_meta = str(task_meta) # Convert to a string representation + except ConnectionError as err: + return make_response( + jsonify({"success": False, "message": str(err)}), 503 + ) except Exception as err: current_app.logger.error(f"Error getting task status: {err}", exc_info=True) return make_response(jsonify({"success": False}), 400) diff --git a/frontend/src/components/ConnectorTreeComponent.tsx b/frontend/src/components/ConnectorTreeComponent.tsx index 185a21c9..03633088 100644 --- a/frontend/src/components/ConnectorTreeComponent.tsx +++ b/frontend/src/components/ConnectorTreeComponent.tsx @@ -6,6 +6,7 @@ import { selectToken } from '../preferences/preferenceSlice'; import { ActiveState } from '../models/misc'; import Chunks from './Chunks'; import ContextMenu, { MenuOption } from './ContextMenu'; +import SkeletonLoader from './SkeletonLoader'; import ConfirmationModal from '../modals/ConfirmationModal'; import userService from '../api/services/userService'; import FileIcon from '../assets/file.svg'; @@ -15,7 +16,7 @@ import ThreeDots from '../assets/three-dots.svg'; import EyeView from '../assets/eye-view.svg'; import SyncIcon from '../assets/sync.svg'; import CheckmarkIcon from '../assets/checkMark2.svg'; -import { useOutsideAlerter } from '../hooks'; +import { useOutsideAlerter, useLoaderState } from '../hooks'; import { Table, TableContainer, @@ -55,7 +56,7 @@ const ConnectorTreeComponent: React.FC = ({ onBackToDocuments, }) => { const { t } = useTranslation(); - const [loading, setLoading] = useState(true); + const [loading, setLoading] = useLoaderState(true, 500); const [error, setError] = useState(null); const [directoryStructure, setDirectoryStructure] = useState(null); @@ -716,7 +717,13 @@ const ConnectorTreeComponent: React.FC = ({ - {renderFileTree(getCurrentDirectory())} + + {loading ? ( + + ) : ( + renderFileTree(getCurrentDirectory()) + )} + diff --git a/frontend/src/components/FileTreeComponent.tsx b/frontend/src/components/FileTreeComponent.tsx index 0e63d621..0e4aeedf 100644 --- a/frontend/src/components/FileTreeComponent.tsx +++ b/frontend/src/components/FileTreeComponent.tsx @@ -5,6 +5,7 @@ import { selectToken } from '../preferences/preferenceSlice'; import { formatBytes } from '../utils/stringUtils'; import Chunks from './Chunks'; import ContextMenu, { MenuOption } from './ContextMenu'; +import SkeletonLoader from './SkeletonLoader'; import userService from '../api/services/userService'; import FileIcon from '../assets/file.svg'; import FolderIcon from '../assets/folder.svg'; @@ -12,7 +13,7 @@ import ArrowLeft from '../assets/arrow-left.svg'; import ThreeDots from '../assets/three-dots.svg'; import EyeView from '../assets/eye-view.svg'; import Trash from '../assets/red-trash.svg'; -import { useOutsideAlerter } from '../hooks'; +import { useOutsideAlerter, useLoaderState } from '../hooks'; import ConfirmationModal from '../modals/ConfirmationModal'; import { Table, @@ -53,7 +54,7 @@ const FileTreeComponent: React.FC = ({ onBackToDocuments, }) => { const { t } = useTranslation(); - const [loading, setLoading] = useState(true); + const [loading, setLoading] = useLoaderState(true, 500); const [error, setError] = useState(null); const [directoryStructure, setDirectoryStructure] = useState(null); @@ -839,7 +840,13 @@ const FileTreeComponent: React.FC = ({ - {renderFileTree(currentDirectory)} + + {loading ? ( + + ) : ( + renderFileTree(currentDirectory) + )} + diff --git a/frontend/src/components/SkeletonLoader.tsx b/frontend/src/components/SkeletonLoader.tsx index 13f6113f..c90d03ba 100644 --- a/frontend/src/components/SkeletonLoader.tsx +++ b/frontend/src/components/SkeletonLoader.tsx @@ -6,7 +6,7 @@ interface SkeletonLoaderProps { | 'default' | 'analysis' | 'logs' - | 'table' + | 'fileTable' | 'chatbot' | 'dropdown' | 'chunkCards' @@ -44,15 +44,15 @@ const SkeletonLoader: React.FC = ({ <> {[...Array(4)].map((_, idx) => ( - + +
+ +
- -
-
@@ -241,7 +241,7 @@ const SkeletonLoader: React.FC = ({ ); const componentMap = { - table: renderTable, + fileTable: renderTable, chatbot: renderChatbot, dropdown: renderDropdown, logs: renderLogs, diff --git a/frontend/src/components/UploadToast.tsx b/frontend/src/components/UploadToast.tsx index dbd23f64..3d1dd03f 100644 --- a/frontend/src/components/UploadToast.tsx +++ b/frontend/src/components/UploadToast.tsx @@ -37,7 +37,7 @@ export default function UploadToast() { case 'completed': return t('modals.uploadDoc.progress.completed'); case 'failed': - return t('attachments.uploadFailed'); + return t('modals.uploadDoc.progress.failed'); default: return t('modals.uploadDoc.progress.preparing'); } diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 25a8fb9e..2307df41 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -262,6 +262,7 @@ "upload": "Upload is in progress", "training": "Upload is in progress", "completed": "Upload completed", + "failed": "Upload failed", "wait": "This may take several minutes", "preparing": "Preparing upload", "tokenLimit": "Over the token limit, please consider uploading smaller document", @@ -424,8 +425,7 @@ }, "attachments": { "attach": "Attach", - "remove": "Remove attachment", - "uploadFailed": "Upload failed" + "remove": "Remove attachment" }, "retry": "Retry" } diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index cedc3232..5ac997ab 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -225,6 +225,7 @@ "upload": "Subida en progreso", "training": "Subida en progreso", "completed": "Subida completada", + "failed": "Error al subir", "wait": "Esto puede tardar varios minutos", "preparing": "Preparando subida", "tokenLimit": "Excede el límite de tokens, considere cargar un documento más pequeño", @@ -387,8 +388,7 @@ }, "attachments": { "attach": "Adjuntar", - "remove": "Eliminar adjunto", - "uploadFailed": "Error al subir" + "remove": "Eliminar adjunto" }, "retry": "Reintentar" } diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index 1072b17c..dec5ed18 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -225,6 +225,7 @@ "upload": "アップロード中", "training": "アップロード中", "completed": "アップロード完了", + "failed": "アップロード失敗", "wait": "数分かかる場合があります", "preparing": "アップロードを準備中", "tokenLimit": "トークン制限を超えています。より小さいドキュメントをアップロードしてください", @@ -387,8 +388,7 @@ }, "attachments": { "attach": "添付", - "remove": "添付ファイルを削除", - "uploadFailed": "アップロード失敗" + "remove": "添付ファイルを削除" }, "retry": "再試行" } diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index 0ca63ec5..c5d4b6c6 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -225,6 +225,7 @@ "upload": "Идет загрузка", "training": "Идет загрузка", "completed": "Загрузка завершена", + "failed": "Ошибка загрузки", "wait": "Это может занять несколько минут", "preparing": "Подготовка загрузки", "tokenLimit": "Превышен лимит токенов, рассмотрите возможность загрузки документа меньшего размера", @@ -387,8 +388,7 @@ }, "attachments": { "attach": "Прикрепить", - "remove": "Удалить вложение", - "uploadFailed": "Ошибка загрузки" + "remove": "Удалить вложение" }, "retry": "Повторить" } diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index 60a0c91d..0e25efbc 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -225,6 +225,7 @@ "upload": "正在上傳", "training": "正在上傳", "completed": "上傳完成", + "failed": "上傳失敗", "wait": "這可能需要幾分鐘", "preparing": "準備上傳", "tokenLimit": "超出令牌限制，請考慮上傳較小的文檔", @@ -387,8 +388,7 @@ }, "attachments": { "attach": "附件", - "remove": "刪除附件", - "uploadFailed": "上傳失敗" + "remove": "刪除附件" }, "retry": "重試" } diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index c3f4ec59..b8713b4e 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -225,6 +225,7 @@ "upload": "正在上传", "training": "正在上传", "completed": "上传完成", + "failed": "上传失败", "wait": "这可能需要几分钟", "preparing": "准备上传", "tokenLimit": "超出令牌限制，请考虑上传较小的文档", @@ -387,8 +388,7 @@ }, "attachments": { "attach": "附件", - "remove": "删除附件", - "uploadFailed": "上传失败" + "remove": "删除附件" }, "retry": "重试" } diff --git a/frontend/src/settings/APIKeys.tsx b/frontend/src/settings/APIKeys.tsx index 577a2bc5..dfeb3c47 100644 --- a/frontend/src/settings/APIKeys.tsx +++ b/frontend/src/settings/APIKeys.tsx @@ -146,7 +146,7 @@ export default function APIKeys() { {loading ? ( - + ) : !apiKeys?.length ? ( response.json()) .then(async (data) => { + if (!data.success && data.message) { + if (timeoutId !== null) { + clearTimeout(timeoutId); + timeoutId = null; + } + handleTaskFailure(clientTaskId, data.message); + return; + } + if (data.status === 'SUCCESS') { if (timeoutId !== null) { clearTimeout(timeoutId); @@ -376,12 +385,12 @@ function Upload({ timeoutId = window.setTimeout(poll, 5000); } }) - .catch(() => { + .catch((error) => { if (timeoutId !== null) { clearTimeout(timeoutId); timeoutId = null; } - handleTaskFailure(clientTaskId); + handleTaskFailure(clientTaskId, error?.message); }); };