Upload: communicate failure, minor frontend updates (#2048)

* (feat:pause-stream) generator exit

* (feat:pause-stream) close request

* (feat:pause-stream) finally close; google anthropic

* (feat:task_status)communicate failure

* (clean:connector) unused routes

* (feat:file-table) missing skeletons

* (fix:apiKeys) build err

---------

Co-authored-by: GH Action - Upstream Sync <action@github.com>

application/api/connector/routes.py

@@ -23,15 +23,9 @@ from application.core.settings import settings
 from application.api import api
 
 
-from application.utils import (
-    check_required_fields
-)
-
-
 from application.parser.connectors.connector_creator import ConnectorCreator
 
 
-
 mongo = MongoDB.get_client()
 db = mongo[settings.MONGO_DB_NAME]
 sources_collection = db["sources"]
@@ -43,185 +37,6 @@ api.add_namespace(connectors_ns)
 
 
 
-@connectors_ns.route("/api/connectors/upload")
-class UploadConnector(Resource):
-    @api.expect(
-        api.model(
-            "ConnectorUploadModel",
-            {
-                "user": fields.String(required=True, description="User ID"),
-                "source": fields.String(
-                    required=True, description="Source type (google_drive, github, etc.)"
-                ),
-                "name": fields.String(required=True, description="Job name"),
-                "data": fields.String(required=True, description="Configuration data"),
-                "repo_url": fields.String(description="GitHub repository URL"),
-            },
-        )
-    )
-    @api.doc(
-        description="Uploads connector source for vectorization",
-    )
-    def post(self):
-        decoded_token = request.decoded_token
-        if not decoded_token:
-            return make_response(jsonify({"success": False}), 401)
-        data = request.form
-        required_fields = ["user", "source", "name", "data"]
-        missing_fields = check_required_fields(data, required_fields)
-        if missing_fields:
-            return missing_fields
-        try:
-            config = json.loads(data["data"])
-            source_data = None
-            sync_frequency = config.get("sync_frequency", "never")
-
-            if data["source"] == "github":
-                source_data = config.get("repo_url")
-            elif data["source"] in ["crawler", "url"]:
-                source_data = config.get("url")
-            elif data["source"] == "reddit":
-                source_data = config
-            elif data["source"] in ConnectorCreator.get_supported_connectors():
-                session_token = config.get("session_token")
-                if not session_token:
-                    return make_response(jsonify({
-                        "success": False,
-                        "error": f"Missing session_token in {data['source']} configuration"
-                    }), 400)
-
-                file_ids = config.get("file_ids", [])
-                if isinstance(file_ids, str):
-                    file_ids = [id.strip() for id in file_ids.split(',') if id.strip()]
-                elif not isinstance(file_ids, list):
-                    file_ids = []
-
-                folder_ids = config.get("folder_ids", [])
-                if isinstance(folder_ids, str):
-                    folder_ids = [id.strip() for id in folder_ids.split(',') if id.strip()]
-                elif not isinstance(folder_ids, list):
-                    folder_ids = []
-
-                config["file_ids"] = file_ids
-                config["folder_ids"] = folder_ids
-
-                task = ingest_connector_task.delay(
-                    job_name=data["name"],
-                    user=decoded_token.get("sub"),
-                    source_type=data["source"],
-                    session_token=session_token,
-                    file_ids=file_ids,
-                    folder_ids=folder_ids,
-                    recursive=config.get("recursive", False),
-                    retriever=config.get("retriever", "classic"),
-                    sync_frequency=sync_frequency
-                )
-                return make_response(jsonify({"success": True, "task_id": task.id}), 200)
-            task = ingest_connector_task.delay(
-                source_data=source_data,
-                job_name=data["name"],
-                user=decoded_token.get("sub"),
-                loader=data["source"],
-                sync_frequency=sync_frequency
-            )
-        except Exception as err:
-            current_app.logger.error(
-                f"Error uploading connector source: {err}", exc_info=True
-            )
-            return make_response(jsonify({"success": False}), 400)
-        return make_response(jsonify({"success": True, "task_id": task.id}), 200)
-
-
-@connectors_ns.route("/api/connectors/task_status")
-class ConnectorTaskStatus(Resource):
-    task_status_model = api.model(
-        "ConnectorTaskStatusModel",
-        {"task_id": fields.String(required=True, description="Task ID")},
-    )
-
-    @api.expect(task_status_model)
-    @api.doc(description="Get connector task status")
-    def get(self):
-        task_id = request.args.get("task_id")
-        if not task_id:
-            return make_response(
-                jsonify({"success": False, "message": "Task ID is required"}), 400
-            )
-        try:
-            from application.celery_init import celery
-
-            task = celery.AsyncResult(task_id)
-            task_meta = task.info
-            print(f"Task status: {task.status}")
-            if not isinstance(
-                task_meta, (dict, list, str, int, float, bool, type(None))
-            ):
-                task_meta = str(task_meta)
-        except Exception as err:
-            current_app.logger.error(f"Error getting task status: {err}", exc_info=True)
-            return make_response(jsonify({"success": False}), 400)
-        return make_response(jsonify({"status": task.status, "result": task_meta}), 200)
-
-
-@connectors_ns.route("/api/connectors/sources")
-class ConnectorSources(Resource):
-    @api.doc(description="Get connector sources")
-    def get(self):
-        decoded_token = request.decoded_token
-        if not decoded_token:
-            return make_response(jsonify({"success": False}), 401)
-        user = decoded_token.get("sub")
-        try:
-            sources = sources_collection.find({"user": user, "type": "connector:file"}).sort("date", -1)
-            connector_sources = []
-            for source in sources:
-                connector_sources.append({
-                    "id": str(source["_id"]),
-                    "name": source.get("name"),
-                    "date": source.get("date"),
-                    "type": source.get("type"),
-                    "source": source.get("source"),
-                    "tokens": source.get("tokens", ""),
-                    "retriever": source.get("retriever", "classic"),
-                    "syncFrequency": source.get("sync_frequency", ""),
-                })
-        except Exception as err:
-            current_app.logger.error(f"Error retrieving connector sources: {err}", exc_info=True)
-            return make_response(jsonify({"success": False}), 400)
-        return make_response(jsonify(connector_sources), 200)
-
-
-@connectors_ns.route("/api/connectors/delete")
-class DeleteConnectorSource(Resource):
-    @api.doc(
-        description="Delete a connector source",
-        params={"source_id": "The source ID to delete"},
-    )
-    def delete(self):
-        decoded_token = request.decoded_token
-        if not decoded_token:
-            return make_response(jsonify({"success": False}), 401)
-        source_id = request.args.get("source_id")
-        if not source_id:
-            return make_response(
-                jsonify({"success": False, "message": "source_id is required"}), 400
-            )
-        try:
-            result = sources_collection.delete_one(
-                {"_id": ObjectId(source_id), "user": decoded_token.get("sub")}
-            )
-            if result.deleted_count == 0:
-                return make_response(
-                    jsonify({"success": False, "message": "Source not found"}), 404
-                )
-        except Exception as err:
-            current_app.logger.error(
-                f"Error deleting connector source: {err}", exc_info=True
-            )
-            return make_response(jsonify({"success": False}), 400)
-        return make_response(jsonify({"success": True}), 200)
-
-
 @connectors_ns.route("/api/connectors/auth")
 class ConnectorAuth(Resource):
     @api.doc(description="Get connector OAuth authorization URL", params={"provider": "Connector provider (e.g., google_drive)"})
@@ -337,27 +152,6 @@ class ConnectorsCallback(Resource):
             return redirect("/api/connectors/callback-status?status=error&message=Authentication+failed.+Please+try+again+and+make+sure+to+grant+all+requested+permissions.")
 
 
-@connectors_ns.route("/api/connectors/refresh")
-class ConnectorRefresh(Resource):
-    @api.expect(api.model("ConnectorRefreshModel", {"provider": fields.String(required=True), "refresh_token": fields.String(required=True)}))
-    @api.doc(description="Refresh connector access token")
-    def post(self):
-        try:
-            data = request.get_json()
-            provider = data.get('provider')
-            refresh_token = data.get('refresh_token')
-
-            if not provider or not refresh_token:
-                return make_response(jsonify({"success": False, "error": "provider and refresh_token are required"}), 400)
-
-            auth = ConnectorCreator.create_auth(provider)
-            token_info = auth.refresh_access_token(refresh_token)
-            return make_response(jsonify({"success": True, "token_info": token_info}), 200)
-        except Exception as e:
-            current_app.logger.error(f"Error refreshing token for connector: {e}")
-            return make_response(jsonify({"success": False, "error": str(e)}), 500)
-
-
 @connectors_ns.route("/api/connectors/files")
 class ConnectorFiles(Resource):
     @api.expect(api.model("ConnectorFilesModel", {

application/api/user/sources/upload.py

@@ -562,10 +562,21 @@ class TaskStatus(Resource):
             task = celery.AsyncResult(task_id)
             task_meta = task.info
             print(f"Task status: {task.status}")
+
+            if task.status == "PENDING":
+                inspect = celery.control.inspect()
+                active_workers = inspect.ping()
+                if not active_workers:
+                    raise ConnectionError("Service unavailable")
+
             if not isinstance(
                 task_meta, (dict, list, str, int, float, bool, type(None))
             ):
                 task_meta = str(task_meta)  # Convert to a string representation
+        except ConnectionError as err:
+            return make_response(
+                jsonify({"success": False, "message": str(err)}), 503
+            )
         except Exception as err:
             current_app.logger.error(f"Error getting task status: {err}", exc_info=True)
             return make_response(jsonify({"success": False}), 400)

frontend/src/components/ConnectorTreeComponent.tsx

@@ -6,6 +6,7 @@ import { selectToken } from '../preferences/preferenceSlice';
 import { ActiveState } from '../models/misc';
 import Chunks from './Chunks';
 import ContextMenu, { MenuOption } from './ContextMenu';
+import SkeletonLoader from './SkeletonLoader';
 import ConfirmationModal from '../modals/ConfirmationModal';
 import userService from '../api/services/userService';
 import FileIcon from '../assets/file.svg';
@@ -15,7 +16,7 @@ import ThreeDots from '../assets/three-dots.svg';
 import EyeView from '../assets/eye-view.svg';
 import SyncIcon from '../assets/sync.svg';
 import CheckmarkIcon from '../assets/checkMark2.svg';
-import { useOutsideAlerter } from '../hooks';
+import { useOutsideAlerter, useLoaderState } from '../hooks';
 import {
   Table,
   TableContainer,
@@ -55,7 +56,7 @@ const ConnectorTreeComponent: React.FC<ConnectorTreeComponentProps> = ({
   onBackToDocuments,
 }) => {
   const { t } = useTranslation();
-  const [loading, setLoading] = useState<boolean>(true);
+  const [loading, setLoading] = useLoaderState(true, 500);
   const [error, setError] = useState<string | null>(null);
   const [directoryStructure, setDirectoryStructure] =
     useState<DirectoryStructure | null>(null);
@@ -716,7 +717,13 @@ const ConnectorTreeComponent: React.FC<ConnectorTreeComponentProps> = ({
                     </TableHeader>
                   </TableRow>
                 </TableHead>
-                <TableBody>{renderFileTree(getCurrentDirectory())}</TableBody>
+                <TableBody>
+                  {loading ? (
+                    <SkeletonLoader component="fileTable" />
+                  ) : (
+                    renderFileTree(getCurrentDirectory())
+                  )}
+                </TableBody>
               </Table>
             </TableContainer>
           </div>

frontend/src/components/FileTreeComponent.tsx

@@ -5,6 +5,7 @@ import { selectToken } from '../preferences/preferenceSlice';
 import { formatBytes } from '../utils/stringUtils';
 import Chunks from './Chunks';
 import ContextMenu, { MenuOption } from './ContextMenu';
+import SkeletonLoader from './SkeletonLoader';
 import userService from '../api/services/userService';
 import FileIcon from '../assets/file.svg';
 import FolderIcon from '../assets/folder.svg';
@@ -12,7 +13,7 @@ import ArrowLeft from '../assets/arrow-left.svg';
 import ThreeDots from '../assets/three-dots.svg';
 import EyeView from '../assets/eye-view.svg';
 import Trash from '../assets/red-trash.svg';
-import { useOutsideAlerter } from '../hooks';
+import { useOutsideAlerter, useLoaderState } from '../hooks';
 import ConfirmationModal from '../modals/ConfirmationModal';
 import {
   Table,
@@ -53,7 +54,7 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
   onBackToDocuments,
 }) => {
   const { t } = useTranslation();
-  const [loading, setLoading] = useState<boolean>(true);
+  const [loading, setLoading] = useLoaderState(true, 500);
   const [error, setError] = useState<string | null>(null);
   const [directoryStructure, setDirectoryStructure] =
     useState<DirectoryStructure | null>(null);
@@ -839,7 +840,13 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
                     </TableHeader>
                   </TableRow>
                 </TableHead>
-                <TableBody>{renderFileTree(currentDirectory)}</TableBody>
+                <TableBody>
+                  {loading ? (
+                    <SkeletonLoader component="fileTable" />
+                  ) : (
+                    renderFileTree(currentDirectory)
+                  )}
+                </TableBody>
               </Table>
             </TableContainer>
           </div>

frontend/src/components/SkeletonLoader.tsx

@@ -6,7 +6,7 @@ interface SkeletonLoaderProps {
     | 'default'
     | 'analysis'
     | 'logs'
-    | 'table'
+    | 'fileTable'
     | 'chatbot'
     | 'dropdown'
     | 'chunkCards'
@@ -44,15 +44,15 @@ const SkeletonLoader: React.FC<SkeletonLoaderProps> = ({
     <>
       {[...Array(4)].map((_, idx) => (
         <tr key={idx} className="animate-pulse">
-          <td className="w-[45%] px-4 py-4">
+          <td className="w-[40%] px-4 py-4">
+            <div className="h-4 w-full rounded-sm bg-gray-300 dark:bg-gray-600"></div>
+          </td>
+          <td className="w-[30%] px-4 py-4">
             <div className="h-4 w-full rounded-sm bg-gray-300 dark:bg-gray-600"></div>
           </td>
           <td className="w-[20%] px-4 py-4">
             <div className="h-4 w-full rounded-sm bg-gray-300 dark:bg-gray-600"></div>
           </td>
-          <td className="w-[25%] px-4 py-4">
-            <div className="h-4 w-full rounded-sm bg-gray-300 dark:bg-gray-600"></div>
-          </td>
           <td className="w-[10%] px-4 py-4">
             <div className="h-4 w-full rounded-sm bg-gray-300 dark:bg-gray-600"></div>
           </td>
@@ -241,7 +241,7 @@ const SkeletonLoader: React.FC<SkeletonLoaderProps> = ({
   );
 
   const componentMap = {
-    table: renderTable,
+    fileTable: renderTable,
     chatbot: renderChatbot,
     dropdown: renderDropdown,
     logs: renderLogs,

frontend/src/components/UploadToast.tsx

@@ -37,7 +37,7 @@ export default function UploadToast() {
       case 'completed':
         return t('modals.uploadDoc.progress.completed');
       case 'failed':
-        return t('attachments.uploadFailed');
+        return t('modals.uploadDoc.progress.failed');
       default:
         return t('modals.uploadDoc.progress.preparing');
     }

frontend/src/locale/en.json

@@ -262,6 +262,7 @@
         "upload": "Upload is in progress",
         "training": "Upload is in progress",
         "completed": "Upload completed",
+        "failed": "Upload failed",
         "wait": "This may take several minutes",
         "preparing": "Preparing upload",
         "tokenLimit": "Over the token limit, please consider uploading smaller document",
@@ -424,8 +425,7 @@
     },
     "attachments": {
       "attach": "Attach",
-      "remove": "Remove attachment",
+      "remove": "Remove attachment"
-      "uploadFailed": "Upload failed"
     },
     "retry": "Retry"
   }

frontend/src/locale/es.json

@@ -225,6 +225,7 @@
         "upload": "Subida en progreso",
         "training": "Subida en progreso",
         "completed": "Subida completada",
+        "failed": "Error al subir",
         "wait": "Esto puede tardar varios minutos",
         "preparing": "Preparando subida",
         "tokenLimit": "Excede el límite de tokens, considere cargar un documento más pequeño",
@@ -387,8 +388,7 @@
     },
     "attachments": {
       "attach": "Adjuntar",
-      "remove": "Eliminar adjunto",
+      "remove": "Eliminar adjunto"
-      "uploadFailed": "Error al subir"
     },
     "retry": "Reintentar"
   }

frontend/src/locale/jp.json

@@ -225,6 +225,7 @@
         "upload": "アップロード中",
         "training": "アップロード中",
         "completed": "アップロード完了",
+        "failed": "アップロード失敗",
         "wait": "数分かかる場合があります",
         "preparing": "アップロードを準備中",
         "tokenLimit": "トークン制限を超えています。より小さいドキュメントをアップロードしてください",
@@ -387,8 +388,7 @@
     },
     "attachments": {
       "attach": "添付",
-      "remove": "添付ファイルを削除",
+      "remove": "添付ファイルを削除"
-      "uploadFailed": "アップロード失敗"
     },
     "retry": "再試行"
   }

frontend/src/locale/ru.json

@@ -225,6 +225,7 @@
         "upload": "Идет загрузка",
         "training": "Идет загрузка",
         "completed": "Загрузка завершена",
+        "failed": "Ошибка загрузки",
         "wait": "Это может занять несколько минут",
         "preparing": "Подготовка загрузки",
         "tokenLimit": "Превышен лимит токенов, рассмотрите возможность загрузки документа меньшего размера",
@@ -387,8 +388,7 @@
     },
     "attachments": {
       "attach": "Прикрепить",
-      "remove": "Удалить вложение",
+      "remove": "Удалить вложение"
-      "uploadFailed": "Ошибка загрузки"
     },
     "retry": "Повторить"
   }

frontend/src/locale/zh-TW.json

@@ -225,6 +225,7 @@
         "upload": "正在上傳",
         "training": "正在上傳",
         "completed": "上傳完成",
+        "failed": "上傳失敗",
         "wait": "這可能需要幾分鐘",
         "preparing": "準備上傳",
         "tokenLimit": "超出令牌限制，請考慮上傳較小的文檔",
@@ -387,8 +388,7 @@
     },
     "attachments": {
       "attach": "附件",
-      "remove": "刪除附件",
+      "remove": "刪除附件"
-      "uploadFailed": "上傳失敗"
     },
     "retry": "重試"
   }

frontend/src/locale/zh.json

@@ -225,6 +225,7 @@
         "upload": "正在上传",
         "training": "正在上传",
         "completed": "上传完成",
+        "failed": "上传失败",
         "wait": "这可能需要几分钟",
         "preparing": "准备上传",
         "tokenLimit": "超出令牌限制，请考虑上传较小的文档",
@@ -387,8 +388,7 @@
     },
     "attachments": {
       "attach": "附件",
-      "remove": "删除附件",
+      "remove": "删除附件"
-      "uploadFailed": "上传失败"
     },
     "retry": "重试"
   }

frontend/src/settings/APIKeys.tsx

@@ -146,7 +146,7 @@ export default function APIKeys() {
                 </thead>
                 <tbody className="dark:divide-silver/40 divide-y divide-gray-300">
                   {loading ? (
-                    <SkeletonLoader component="table" />
+                    <SkeletonLoader component="fileTable" />
                   ) : !apiKeys?.length ? (
                     <tr>
                       <td

frontend/src/upload/Upload.tsx

@@ -291,12 +291,12 @@ function Upload({
           id: clientTaskId,
           updates: {
             status: 'failed',
-            errorMessage: errorMessage || t('attachments.uploadFailed'),
+            errorMessage: errorMessage,
           },
         }),
       );
     },
-    [dispatch, t],
+    [dispatch],
   );
 
   const trackTraining = useCallback(
@@ -308,6 +308,15 @@ function Upload({
           .getTaskStatus(backendTaskId, null)
           .then((response) => response.json())
           .then(async (data) => {
+            if (!data.success && data.message) {
+              if (timeoutId !== null) {
+                clearTimeout(timeoutId);
+                timeoutId = null;
+              }
+              handleTaskFailure(clientTaskId, data.message);
+              return;
+            }
+
             if (data.status === 'SUCCESS') {
               if (timeoutId !== null) {
                 clearTimeout(timeoutId);
@@ -376,12 +385,12 @@ function Upload({
               timeoutId = window.setTimeout(poll, 5000);
             }
           })
-          .catch(() => {
+          .catch((error) => {
             if (timeoutId !== null) {
               clearTimeout(timeoutId);
               timeoutId = null;
             }
-            handleTaskFailure(clientTaskId);
+            handleTaskFailure(clientTaskId, error?.message);
           });
       };