LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2026-01-20 14:00:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

multiple-file missing task ingest

Browse Source 
restore upload to pack all file together instead one by one
This commit is contained in:
fadingNA
2024-11-10 02:59:09 -05:00
parent d5cce88108
commit 8a225e279f
2 changed files with 65 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
application/api/user/routes.py
View File

@@ -315,7 +315,7 @@ class UploadFile(Resource):
for file in files:
filename = secure_filename(file.filename)
file.save(os.path.join(temp_dir, filename))
print(f"Saved file: {filename}")
zip_path = shutil.make_archive(
base_name=os.path.join(save_dir, job_name),
format="zip",
@@ -323,6 +323,26 @@ class UploadFile(Resource):
)
final_filename = os.path.basename(zip_path)
shutil.rmtree(temp_dir)
task = ingest.delay(
settings.UPLOAD_FOLDER,
[
".rst",
".md",
".pdf",
".txt",
".docx",
".csv",
".epub",
".html",
".mdx",
".json",
".xlsx",
".pptx",
],
job_name,
final_filename,
user,
)
else:
file = files[0]
final_filename = secure_filename(file.filename)
@@ -349,9 +369,10 @@ class UploadFile(Resource):
final_filename,
user,
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
except Exception as err:
print(f"Error: {err}")
return make_response(jsonify({"success": False, "error": str(err)}), 400)
return make_response(jsonify({"success": True, "task_id": task.id}), 200)
@@ -422,6 +443,11 @@ class TaskStatus(Resource):
task = celery.AsyncResult(task_id)
task_meta = task.info
print(f"Task status: {task.status}")
if not isinstance(
task_meta, (dict, list, str, int, float, bool, type(None))
):
task_meta = str(task_meta) # Convert to a string representation
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)
@@ -433,8 +459,8 @@ class CombinedJson(Resource):
@api.doc(description="Provide JSON file with combined available indexes")
def get(self):
user = "local"
sort_field = request.args.get('sort', 'date') # Default to 'date'
sort_order = request.args.get('order', "desc") # Default to 'desc'
sort_field = request.args.get("sort", "date") # Default to 'date'
sort_order = request.args.get("order", "desc") # Default to 'desc'
data = [
{
"name": "default",
@@ -447,7 +473,9 @@ class CombinedJson(Resource):
]
try:
for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1):
for index in sources_collection.find({"user": user}).sort(
sort_field, 1 if sort_order == "asc" else -1
):
data.append(
{
"id": str(index["_id"]),
@@ -1674,7 +1702,9 @@ class TextToSpeech(Resource):
tts_model = api.model(
"TextToSpeechModel",
{
"text": fields.String(required=True, description="Text to be synthesized as audio"),
"text": fields.String(
required=True, description="Text to be synthesized as audio"
),
},
)
@@ -1686,8 +1716,15 @@ class TextToSpeech(Resource):
try:
tts_instance = GoogleTTS()
audio_base64, detected_language = tts_instance.text_to_speech(text)
return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200)
return make_response(
jsonify(
{
"success": True,
"audio_base64": audio_base64,
"lang": detected_language,
}
),
200,
)
except Exception as err:
return make_response(jsonify({"success": False, "error": str(err)}), 400)

69
frontend/src/upload/Upload.tsx
View File

@@ -243,59 +243,26 @@ function Upload({
const doNothing = () => undefined;
const uploadFile = () => {
if (files.length === 0) {
console.error('No files to upload.');
return;
}
const apiHost = import.meta.env.VITE_API_HOST;
const uploadNextFile = (index: number) => {
if (index >= files.length) {
console.log('All files have been uploaded.');
return;
}
const file = files[index];
const formData = new FormData();
const formData = new FormData();
files.forEach((file) => {
formData.append('file', file);
formData.append('name', docName);
formData.append('user', 'local');
const xhr = new XMLHttpRequest();
xhr.upload.addEventListener('progress', (event) => {
const progress = +((event.loaded / event.total) * 100).toFixed(2);
console.log(`File ${index + 1}/${files.length} progress: ${progress}%`);
setProgress({ type: 'UPLOAD', percentage: progress });
});
xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) {
const { task_id } = JSON.parse(xhr.responseText);
console.log(`
File ${index + 1}/${files.length} uploaded successfully. Task ID: ${task_id}`);
setTimeoutRef.current = setTimeout(() => {
setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id });
}, 3000);
uploadNextFile(index + 1);
} else {
console.error(
`
File ${index + 1}/${files.length} upload failed with status ${xhr.status}. Response:`,
xhr.responseText,
);
}
};
xhr.onerror = () => {
console.error(`
File ${index + 1}/${files.length} upload encountered an error.`);
};
xhr.open('POST', `${apiHost + '/api/upload'}`);
xhr.send(formData);
});
formData.append('name', docName);
formData.append('user', 'local');
const apiHost = import.meta.env.VITE_API_HOST;
const xhr = new XMLHttpRequest();
xhr.upload.addEventListener('progress', (event) => {
const progress = +((event.loaded / event.total) * 100).toFixed(2);
setProgress({ type: 'UPLOAD', percentage: progress });
});
xhr.onload = () => {
const { task_id } = JSON.parse(xhr.responseText);
setTimeoutRef.current = setTimeout(() => {
setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id });
}, 3000);
};
uploadNextFile(0);
xhr.open('POST', `${apiHost + '/api/upload'}`);
xhr.send(formData);
};
const uploadRemote = () => {