From 8a225e279fcfa57d2205cecf763a1a46a967db4f Mon Sep 17 00:00:00 2001 From: fadingNA Date: Sun, 10 Nov 2024 02:59:09 -0500 Subject: [PATCH] multiple-file missing task ingest restore upload to pack all file together instead one by one --- application/api/user/routes.py | 57 +++++++++++++++++++++++----- frontend/src/upload/Upload.tsx | 69 +++++++++------------------------- 2 files changed, 65 insertions(+), 61 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 8e62683e..0868333e 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -315,7 +315,7 @@ class UploadFile(Resource): for file in files: filename = secure_filename(file.filename) file.save(os.path.join(temp_dir, filename)) - + print(f"Saved file: {filename}") zip_path = shutil.make_archive( base_name=os.path.join(save_dir, job_name), format="zip", @@ -323,6 +323,26 @@ class UploadFile(Resource): ) final_filename = os.path.basename(zip_path) shutil.rmtree(temp_dir) + task = ingest.delay( + settings.UPLOAD_FOLDER, + [ + ".rst", + ".md", + ".pdf", + ".txt", + ".docx", + ".csv", + ".epub", + ".html", + ".mdx", + ".json", + ".xlsx", + ".pptx", + ], + job_name, + final_filename, + user, + ) else: file = files[0] final_filename = secure_filename(file.filename) @@ -349,9 +369,10 @@ class UploadFile(Resource): final_filename, user, ) - except Exception as err: - return make_response(jsonify({"success": False, "error": str(err)}), 400) + except Exception as err: + print(f"Error: {err}") + return make_response(jsonify({"success": False, "error": str(err)}), 400) return make_response(jsonify({"success": True, "task_id": task.id}), 200) @@ -422,6 +443,11 @@ class TaskStatus(Resource): task = celery.AsyncResult(task_id) task_meta = task.info + print(f"Task status: {task.status}") + if not isinstance( + task_meta, (dict, list, str, int, float, bool, type(None)) + ): + task_meta = str(task_meta) # Convert to a string representation except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) @@ -433,8 +459,8 @@ class CombinedJson(Resource): @api.doc(description="Provide JSON file with combined available indexes") def get(self): user = "local" - sort_field = request.args.get('sort', 'date') # Default to 'date' - sort_order = request.args.get('order', "desc") # Default to 'desc' + sort_field = request.args.get("sort", "date") # Default to 'date' + sort_order = request.args.get("order", "desc") # Default to 'desc' data = [ { "name": "default", @@ -447,7 +473,9 @@ class CombinedJson(Resource): ] try: - for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1): + for index in sources_collection.find({"user": user}).sort( + sort_field, 1 if sort_order == "asc" else -1 + ): data.append( { "id": str(index["_id"]), @@ -1674,7 +1702,9 @@ class TextToSpeech(Resource): tts_model = api.model( "TextToSpeechModel", { - "text": fields.String(required=True, description="Text to be synthesized as audio"), + "text": fields.String( + required=True, description="Text to be synthesized as audio" + ), }, ) @@ -1686,8 +1716,15 @@ class TextToSpeech(Resource): try: tts_instance = GoogleTTS() audio_base64, detected_language = tts_instance.text_to_speech(text) - return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200) + return make_response( + jsonify( + { + "success": True, + "audio_base64": audio_base64, + "lang": detected_language, + } + ), + 200, + ) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - - diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 7ffb94d1..81ce9f2b 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -243,59 +243,26 @@ function Upload({ const doNothing = () => undefined; const uploadFile = () => { - if (files.length === 0) { - console.error('No files to upload.'); - return; - } - - const apiHost = import.meta.env.VITE_API_HOST; - - const uploadNextFile = (index: number) => { - if (index >= files.length) { - console.log('All files have been uploaded.'); - return; - } - - const file = files[index]; - const formData = new FormData(); + const formData = new FormData(); + files.forEach((file) => { formData.append('file', file); - formData.append('name', docName); - formData.append('user', 'local'); - - const xhr = new XMLHttpRequest(); - xhr.upload.addEventListener('progress', (event) => { - const progress = +((event.loaded / event.total) * 100).toFixed(2); - console.log(`File ${index + 1}/${files.length} progress: ${progress}%`); - setProgress({ type: 'UPLOAD', percentage: progress }); - }); - - xhr.onload = () => { - if (xhr.status >= 200 && xhr.status < 300) { - const { task_id } = JSON.parse(xhr.responseText); - console.log(` - File ${index + 1}/${files.length} uploaded successfully. Task ID: ${task_id}`); - setTimeoutRef.current = setTimeout(() => { - setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id }); - }, 3000); - uploadNextFile(index + 1); - } else { - console.error( - ` - File ${index + 1}/${files.length} upload failed with status ${xhr.status}. Response:`, - xhr.responseText, - ); - } - }; - - xhr.onerror = () => { - console.error(` - File ${index + 1}/${files.length} upload encountered an error.`); - }; - - xhr.open('POST', `${apiHost + '/api/upload'}`); - xhr.send(formData); + }); + formData.append('name', docName); + formData.append('user', 'local'); + const apiHost = import.meta.env.VITE_API_HOST; + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener('progress', (event) => { + const progress = +((event.loaded / event.total) * 100).toFixed(2); + setProgress({ type: 'UPLOAD', percentage: progress }); + }); + xhr.onload = () => { + const { task_id } = JSON.parse(xhr.responseText); + setTimeoutRef.current = setTimeout(() => { + setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id }); + }, 3000); }; - uploadNextFile(0); + xhr.open('POST', `${apiHost + '/api/upload'}`); + xhr.send(formData); }; const uploadRemote = () => {