Merge pull request #1873 from ManishMadan2882/main

Sources are the new Docs

application/api/internal/routes.py

@@ -1,5 +1,6 @@
 import os
 import datetime
+import json
 from flask import Blueprint, request, send_from_directory
 from werkzeug.utils import secure_filename
 from bson.objectid import ObjectId
@@ -48,7 +49,17 @@ def upload_index_files():
     remote_data = request.form["remote_data"] if "remote_data" in request.form else None
     sync_frequency = request.form["sync_frequency"] if "sync_frequency" in request.form else None
     
-    original_file_path = request.form.get("original_file_path")
+    file_path = request.form.get("file_path")
+    directory_structure = request.form.get("directory_structure")
+    
+    if directory_structure:
+        try:
+            directory_structure = json.loads(directory_structure)
+        except Exception:
+            logger.error("Error parsing directory_structure")
+            directory_structure = {}
+    else:
+        directory_structure = {}
 
     storage = StorageCreator.get_storage()
     index_base_path = f"indexes/{id}"
@@ -66,10 +77,13 @@ def upload_index_files():
         file_pkl = request.files["file_pkl"]
         if file_pkl.filename == "":
             return {"status": "no file name"}
-        
+
         # Save index files to storage
-        storage.save_file(file_faiss, f"{index_base_path}/index.faiss")
-        storage.save_file(file_pkl, f"{index_base_path}/index.pkl")
+        faiss_storage_path = f"{index_base_path}/index.faiss"
+        pkl_storage_path = f"{index_base_path}/index.pkl"
+        storage.save_file(file_faiss, faiss_storage_path)
+        storage.save_file(file_pkl, pkl_storage_path)
+
 
     existing_entry = sources_collection.find_one({"_id": ObjectId(id)})
     if existing_entry:
@@ -87,7 +101,8 @@ def upload_index_files():
                     "retriever": retriever,
                     "remote_data": remote_data,
                     "sync_frequency": sync_frequency,
-                    "file_path": original_file_path,
+                    "file_path": file_path,
+                    "directory_structure": directory_structure,
                 }
             },
         )
@@ -105,7 +120,8 @@ def upload_index_files():
                 "retriever": retriever,
                 "remote_data": remote_data,
                 "sync_frequency": sync_frequency,
-                "file_path": original_file_path,
+                "file_path": file_path,
+                "directory_structure": directory_structure,
             }
         )
     return {"status": "ok"}

application/api/user/routes.py

@@ -3,11 +3,11 @@ import json
 import math
 import os
 import secrets
-import shutil
 import uuid
 from functools import wraps
 from typing import Optional, Tuple
-
+import tempfile
+import zipfile
 from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
@@ -44,6 +44,7 @@ from application.utils import (
     validate_function_name,
     validate_required_fields,
 )
+from application.utils import num_tokens_from_string
 from application.vectorstore.vector_creator import VectorCreator
 
 storage = StorageCreator.get_storage()
@@ -474,7 +475,7 @@ class DeleteByIds(Resource):
 @user_ns.route("/api/delete_old")
 class DeleteOldIndexes(Resource):
     @api.doc(
-        description="Deletes old indexes",
+        description="Deletes old indexes and associated files",
         params={"source_id": "The source ID to delete"},
     )
     def get(self):
@@ -491,21 +492,40 @@ class DeleteOldIndexes(Resource):
         )
         if not doc:
             return make_response(jsonify({"status": "not found"}), 404)
+        
+        storage = StorageCreator.get_storage()
+        
         try:
+            # Delete vector index
             if settings.VECTOR_STORE == "faiss":
-                shutil.rmtree(os.path.join(current_dir, "indexes", str(doc["_id"])))
+                index_path = f"indexes/{str(doc['_id'])}"
+                if storage.file_exists(f"{index_path}/index.faiss"):
+                    storage.delete_file(f"{index_path}/index.faiss")
+                if storage.file_exists(f"{index_path}/index.pkl"):
+                    storage.delete_file(f"{index_path}/index.pkl")
             else:
                 vectorstore = VectorCreator.create_vectorstore(
                     settings.VECTOR_STORE, source_id=str(doc["_id"])
                 )
                 vectorstore.delete_index()
+                
+            if "file_path" in doc and doc["file_path"]:
+                file_path = doc["file_path"]
+                if storage.is_directory(file_path):
+                    files = storage.list_files(file_path)
+                    for f in files:
+                        storage.delete_file(f)
+                else:
+                    storage.delete_file(file_path)
+                    
         except FileNotFoundError:
             pass
         except Exception as err:
             current_app.logger.error(
-                f"Error deleting old indexes: {err}", exc_info=True
+                f"Error deleting files and indexes: {err}", exc_info=True
             )
             return make_response(jsonify({"success": False}), 400)
+            
         sources_collection.delete_one({"_id": ObjectId(source_id)})
         return make_response(jsonify({"success": True}), 200)
 
@@ -549,146 +569,276 @@ class UploadFile(Resource):
         # Create safe versions for filesystem operations
         safe_user = safe_filename(user)
         dir_name = safe_filename(job_name)
+        base_path = f"{settings.UPLOAD_FOLDER}/{safe_user}/{dir_name}"
 
         try:
             storage = StorageCreator.get_storage()
-            base_path = f"{settings.UPLOAD_FOLDER}/{safe_user}/{dir_name}"
-
-            if len(files) > 1:
-                temp_files = []
-                for file in files:
-                    filename = safe_filename(file.filename)
-                    temp_path = f"{base_path}/temp/{filename}"
-                    storage.save_file(file, temp_path)
-                    temp_files.append(temp_path)
-                    print(f"Saved file: {filename}")
-                zip_filename = f"{dir_name}.zip"
-                zip_path = f"{base_path}/{zip_filename}"
-                zip_temp_path = None
-
-                def create_zip_archive(temp_paths, dir_name, storage):
-                    import tempfile
-
-                    with tempfile.NamedTemporaryFile(
-                        delete=False, suffix=".zip"
-                    ) as temp_zip_file:
-                        zip_output_path = temp_zip_file.name
-                    with tempfile.TemporaryDirectory() as stage_dir:
-                        for path in temp_paths:
-                            try:
-                                file_data = storage.get_file(path)
-                                with open(
-                                    os.path.join(stage_dir, os.path.basename(path)),
-                                    "wb",
-                                ) as f:
-                                    f.write(file_data.read())
-                            except Exception as e:
-                                current_app.logger.error(
-                                    f"Error processing file {path} for zipping: {e}",
-                                    exc_info=True,
-                                )
-                                if os.path.exists(zip_output_path):
-                                    os.remove(zip_output_path)
-                                raise
+            
+            
+            for file in files:
+                original_filename = file.filename
+                safe_file = safe_filename(original_filename)
+                
+                with tempfile.TemporaryDirectory() as temp_dir:
+                    temp_file_path = os.path.join(temp_dir, safe_file)
+                    file.save(temp_file_path)
+                    
+                    if zipfile.is_zipfile(temp_file_path):
                         try:
-                            shutil.make_archive(
-                                base_name=zip_output_path.replace(".zip", ""),
-                                format="zip",
-                                root_dir=stage_dir,
-                            )
+                            with zipfile.ZipFile(temp_file_path, 'r') as zip_ref:
+                                zip_ref.extractall(path=temp_dir)
+                                
+                                # Walk through extracted files and upload them
+                                for root, _, files in os.walk(temp_dir):
+                                    for extracted_file in files:
+                                        if os.path.join(root, extracted_file) == temp_file_path:
+                                            continue
+                                            
+                                        rel_path = os.path.relpath(os.path.join(root, extracted_file), temp_dir)
+                                        storage_path = f"{base_path}/{rel_path}"
+                                        
+                                        with open(os.path.join(root, extracted_file), 'rb') as f:
+                                            storage.save_file(f, storage_path)
                         except Exception as e:
-                            current_app.logger.error(
-                                f"Error creating zip archive: {e}", exc_info=True
-                            )
-                            if os.path.exists(zip_output_path):
-                                os.remove(zip_output_path)
-                            raise
-                    return zip_output_path
-
-                try:
-                    zip_temp_path = create_zip_archive(temp_files, dir_name, storage)
-                    with open(zip_temp_path, "rb") as zip_file:
-                        storage.save_file(zip_file, zip_path)
-                    task = ingest.delay(
-                        settings.UPLOAD_FOLDER,
-                        [
-                            ".rst",
-                            ".md",
-                            ".pdf",
-                            ".txt",
-                            ".docx",
-                            ".csv",
-                            ".epub",
-                            ".html",
-                            ".mdx",
-                            ".json",
-                            ".xlsx",
-                            ".pptx",
-                            ".png",
-                            ".jpg",
-                            ".jpeg",
-                        ],
-                        job_name,
-                        zip_filename,
-                        user,
-                        dir_name,
-                        safe_user,
-                    )
-                finally:
-                    # Clean up temporary files
-
-                    for temp_path in temp_files:
-                        try:
-                            storage.delete_file(temp_path)
-                        except Exception as e:
-                            current_app.logger.error(
-                                f"Error deleting temporary file {temp_path}: {e}",
-                                exc_info=True,
-                            )
-                    # Clean up the zip file if it was created
-
-                    if zip_temp_path and os.path.exists(zip_temp_path):
-                        os.remove(zip_temp_path)
-            else:  # Keep this else block for single file upload
-                # For single file
-
-                file = files[0]
-                filename = safe_filename(file.filename)
-                file_path = f"{base_path}/{filename}"
-
-                storage.save_file(file, file_path)
-
-                task = ingest.delay(
-                    settings.UPLOAD_FOLDER,
-                    [
-                        ".rst",
-                        ".md",
-                        ".pdf",
-                        ".txt",
-                        ".docx",
-                        ".csv",
-                        ".epub",
-                        ".html",
-                        ".mdx",
-                        ".json",
-                        ".xlsx",
-                        ".pptx",
-                        ".png",
-                        ".jpg",
-                        ".jpeg",
-                    ],
-                    job_name,
-                    filename,  # Corrected variable for single-file case
-                    user,
-                    dir_name,
-                    safe_user,
-                )
+                            current_app.logger.error(f"Error extracting zip: {e}", exc_info=True)
+                            # If zip extraction fails, save the original zip file
+                            file_path = f"{base_path}/{safe_file}"
+                            with open(temp_file_path, 'rb') as f:
+                                storage.save_file(f, file_path)
+                    else:
+                        # For non-zip files, save directly
+                        file_path = f"{base_path}/{safe_file}"
+                        with open(temp_file_path, 'rb') as f:
+                            storage.save_file(f, file_path)
+            
+            task = ingest.delay(
+                settings.UPLOAD_FOLDER,
+                [
+                    ".rst", ".md", ".pdf", ".txt", ".docx", ".csv", ".epub",
+                    ".html", ".mdx", ".json", ".xlsx", ".pptx", ".png",
+                    ".jpg", ".jpeg",
+                ],
+                job_name,
+                user,
+                file_path=base_path,
+                filename=dir_name
+            )
         except Exception as err:
             current_app.logger.error(f"Error uploading file: {err}", exc_info=True)
             return make_response(jsonify({"success": False}), 400)
         return make_response(jsonify({"success": True, "task_id": task.id}), 200)
 
 
+@user_ns.route("/api/manage_source_files")
+class ManageSourceFiles(Resource):
+    @api.expect(
+        api.model(
+            "ManageSourceFilesModel",
+            {
+                "source_id": fields.String(required=True, description="Source ID to modify"),
+                "operation": fields.String(required=True, description="Operation: 'add', 'remove', or 'remove_directory'"),
+                "file_paths": fields.List(fields.String, required=False, description="File paths to remove (for remove operation)"),
+                "directory_path": fields.String(required=False, description="Directory path to remove (for remove_directory operation)"),
+                "file": fields.Raw(required=False, description="Files to add (for add operation)"),
+                "parent_dir": fields.String(required=False, description="Parent directory path relative to source root"),
+            },
+        )
+    )
+    @api.doc(
+        description="Add files, remove files, or remove directories from an existing source",
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False, "message": "Unauthorized"}), 401)
+
+        user = decoded_token.get("sub")
+        source_id = request.form.get("source_id")
+        operation = request.form.get("operation")
+
+        if not source_id or not operation:
+            return make_response(
+                jsonify({"success": False, "message": "source_id and operation are required"}), 400
+            )
+
+        if operation not in ["add", "remove", "remove_directory"]:
+            return make_response(
+                jsonify({"success": False, "message": "operation must be 'add', 'remove', or 'remove_directory'"}), 400
+            )
+
+        try:
+            ObjectId(source_id)
+        except Exception:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid source ID format"}), 400
+            )
+
+        try:
+            source = sources_collection.find_one({"_id": ObjectId(source_id), "user": user})
+            if not source:
+                return make_response(
+                    jsonify({"success": False, "message": "Source not found or access denied"}), 404
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error finding source: {err}", exc_info=True)
+            return make_response(jsonify({"success": False, "message": "Database error"}), 500)
+
+        try:
+            storage = StorageCreator.get_storage()
+            source_file_path = source.get("file_path", "")
+            parent_dir = request.form.get("parent_dir", "") 
+            
+            if parent_dir and (parent_dir.startswith("/") or ".." in parent_dir):
+                return make_response(
+                    jsonify({"success": False, "message": "Invalid parent directory path"}), 400
+                )
+
+            if operation == "add":
+                files = request.files.getlist("file")
+                if not files or all(file.filename == "" for file in files):
+                    return make_response(
+                        jsonify({"success": False, "message": "No files provided for add operation"}), 400
+                    )
+
+                added_files = []
+                
+                target_dir = source_file_path
+                if parent_dir:
+                    target_dir = f"{source_file_path}/{parent_dir}"
+
+                for file in files:
+                    if file.filename:
+                        safe_filename_str = safe_filename(file.filename)
+                        file_path = f"{target_dir}/{safe_filename_str}"
+
+                        # Save file to storage
+                        storage.save_file(file, file_path)
+                        added_files.append(safe_filename_str)
+
+                # Trigger re-ingestion pipeline
+                from application.api.user.tasks import reingest_source_task
+
+                task = reingest_source_task.delay(source_id=source_id, user=user)
+
+                return make_response(jsonify({
+                    "success": True,
+                    "message": f"Added {len(added_files)} files",
+                    "added_files": added_files,
+                    "parent_dir": parent_dir,
+                    "reingest_task_id": task.id
+                }), 200)
+
+            elif operation == "remove":
+                file_paths_str = request.form.get("file_paths")
+                if not file_paths_str:
+                    return make_response(
+                        jsonify({"success": False, "message": "file_paths required for remove operation"}), 400
+                    )
+
+                try:
+                    file_paths = json.loads(file_paths_str) if isinstance(file_paths_str, str) else file_paths_str
+                except Exception:
+                    return make_response(
+                        jsonify({"success": False, "message": "Invalid file_paths format"}), 400
+                    )
+
+                # Remove files from storage and directory structure
+                removed_files = []
+                for file_path in file_paths:
+                    full_path = f"{source_file_path}/{file_path}"
+
+                    # Remove from storage
+                    if storage.file_exists(full_path):
+                        storage.delete_file(full_path)
+                        removed_files.append(file_path)
+
+                # Trigger re-ingestion pipeline
+                from application.api.user.tasks import reingest_source_task
+
+                task = reingest_source_task.delay(source_id=source_id, user=user)
+
+                return make_response(jsonify({
+                    "success": True,
+                    "message": f"Removed {len(removed_files)} files",
+                    "removed_files": removed_files,
+                    "reingest_task_id": task.id
+                }), 200)
+
+            elif operation == "remove_directory":
+                directory_path = request.form.get("directory_path")
+                if not directory_path:
+                    return make_response(
+                        jsonify({"success": False, "message": "directory_path required for remove_directory operation"}), 400
+                    )
+
+                # Validate directory path (prevent path traversal)
+                if directory_path.startswith("/") or ".." in directory_path:
+                    current_app.logger.warning(
+                        f"Invalid directory path attempted for removal. "
+                        f"User: {user}, Source ID: {source_id}, Directory path: {directory_path}"
+                    )
+                    return make_response(
+                        jsonify({"success": False, "message": "Invalid directory path"}), 400
+                    )
+
+                full_directory_path = f"{source_file_path}/{directory_path}" if directory_path else source_file_path
+
+                if not storage.is_directory(full_directory_path):
+                    current_app.logger.warning(
+                        f"Directory not found or is not a directory for removal. "
+                        f"User: {user}, Source ID: {source_id}, Directory path: {directory_path}, "
+                        f"Full path: {full_directory_path}"
+                    )
+                    return make_response(
+                        jsonify({"success": False, "message": "Directory not found or is not a directory"}), 404
+                    )
+
+                success = storage.remove_directory(full_directory_path)
+
+                if not success:
+                    current_app.logger.error(
+                        f"Failed to remove directory from storage. "
+                        f"User: {user}, Source ID: {source_id}, Directory path: {directory_path}, "
+                        f"Full path: {full_directory_path}"
+                    )
+                    return make_response(
+                        jsonify({"success": False, "message": "Failed to remove directory"}), 500
+                    )
+
+                current_app.logger.info(
+                    f"Successfully removed directory. "
+                    f"User: {user}, Source ID: {source_id}, Directory path: {directory_path}, "
+                    f"Full path: {full_directory_path}"
+                )
+
+                # Trigger re-ingestion pipeline
+                from application.api.user.tasks import reingest_source_task
+
+                task = reingest_source_task.delay(source_id=source_id, user=user)
+
+                return make_response(jsonify({
+                    "success": True,
+                    "message": f"Successfully removed directory: {directory_path}",
+                    "removed_directory": directory_path,
+                    "reingest_task_id": task.id
+                }), 200)
+
+        except Exception as err:
+            error_context = f"operation={operation}, user={user}, source_id={source_id}"
+            if operation == "remove_directory":
+                directory_path = request.form.get("directory_path", "")
+                error_context += f", directory_path={directory_path}"
+            elif operation == "remove":
+                file_paths_str = request.form.get("file_paths", "")
+                error_context += f", file_paths={file_paths_str}"
+            elif operation == "add":
+                parent_dir = request.form.get("parent_dir", "")
+                error_context += f", parent_dir={parent_dir}"
+
+            current_app.logger.error(f"Error managing source files: {err} ({error_context})", exc_info=True)
+            return make_response(jsonify({"success": False, "message": "Operation failed"}), 500)
+
+
 @user_ns.route("/api/remote")
 class UploadRemote(Resource):
     @api.expect(
@@ -834,6 +984,7 @@ class PaginatedSources(Resource):
                     "tokens": doc.get("tokens", ""),
                     "retriever": doc.get("retriever", "classic"),
                     "syncFrequency": doc.get("sync_frequency", ""),
+                    "isNested": bool(doc.get("directory_structure"))
                 }
                 paginated_docs.append(doc_data)
             response = {
@@ -881,6 +1032,7 @@ class CombinedJson(Resource):
                         "tokens": index.get("tokens", ""),
                         "retriever": index.get("retriever", "classic"),
                         "syncFrequency": index.get("sync_frequency", ""),
+                        "is_nested": bool(index.get("directory_structure"))
                     }
                 )
         except Exception as err:
@@ -3374,8 +3526,14 @@ class DeleteTool(Resource):
 @user_ns.route("/api/get_chunks")
 class GetChunks(Resource):
     @api.doc(
-        description="Retrieves all chunks associated with a document",
-        params={"id": "The document ID"},
+        description="Retrieves chunks from a document, optionally filtered by file path and search term",
+        params={
+            "id": "The document ID",
+            "page": "Page number for pagination",
+            "per_page": "Number of chunks per page",
+            "path": "Optional: Filter chunks by relative file path",
+            "search": "Optional: Search term to filter chunks by title or content"
+        },
     )
     def get(self):
         decoded_token = request.decoded_token
@@ -3385,6 +3543,8 @@ class GetChunks(Resource):
         doc_id = request.args.get("id")
         page = int(request.args.get("page", 1))
         per_page = int(request.args.get("per_page", 10))
+        path = request.args.get("path")
+        search_term = request.args.get("search", "").strip().lower()
 
         if not ObjectId.is_valid(doc_id):
             return make_response(jsonify({"error": "Invalid doc_id"}), 400)
@@ -3396,6 +3556,30 @@ class GetChunks(Resource):
         try:
             store = get_vector_store(doc_id)
             chunks = store.get_chunks()
+            
+            filtered_chunks = []
+            for chunk in chunks:
+                metadata = chunk.get("metadata", {})
+
+                # Filter by path if provided
+                if path:
+                    chunk_source = metadata.get("source", "")
+                    # Check if the chunk's source matches the requested path
+                    if not chunk_source or not chunk_source.endswith(path):
+                        continue
+
+                # Filter by search term if provided
+                if search_term:
+                    text_match = search_term in chunk.get("text", "").lower()
+                    title_match = search_term in metadata.get("title", "").lower()
+
+                    if not (text_match or title_match):
+                        continue
+
+                filtered_chunks.append(chunk)
+            
+            chunks = filtered_chunks
+            
             total_chunks = len(chunks)
             start = (page - 1) * per_page
             end = start + per_page
@@ -3408,6 +3592,8 @@ class GetChunks(Resource):
                         "per_page": per_page,
                         "total": total_chunks,
                         "chunks": paginated_chunks,
+                        "path": path if path else None,
+                        "search": search_term if search_term else None
                     }
                 ),
                 200,
@@ -3416,7 +3602,6 @@ class GetChunks(Resource):
             current_app.logger.error(f"Error getting chunks: {e}", exc_info=True)
             return make_response(jsonify({"success": False}), 500)
 
-
 @user_ns.route("/api/add_chunk")
 class AddChunk(Resource):
     @api.expect(
@@ -3448,6 +3633,8 @@ class AddChunk(Resource):
         doc_id = data.get("id")
         text = data.get("text")
         metadata = data.get("metadata", {})
+        token_count = num_tokens_from_string(text)
+        metadata["token_count"] = token_count
 
         if not ObjectId.is_valid(doc_id):
             return make_response(jsonify({"error": "Invalid doc_id"}), 400)
@@ -3544,6 +3731,12 @@ class UpdateChunk(Resource):
         text = data.get("text")
         metadata = data.get("metadata")
 
+        if text is not None:
+            token_count = num_tokens_from_string(text)
+            if metadata is None:
+                metadata = {}
+            metadata["token_count"] = token_count
+
         if not ObjectId.is_valid(doc_id):
             return make_response(jsonify({"error": "Invalid doc_id"}), 400)
         doc = sources_collection.find_one({"_id": ObjectId(doc_id), "user": user})
@@ -3553,31 +3746,45 @@ class UpdateChunk(Resource):
             )
         try:
             store = get_vector_store(doc_id)
+
             chunks = store.get_chunks()
             existing_chunk = next((c for c in chunks if c["doc_id"] == chunk_id), None)
             if not existing_chunk:
                 return make_response(jsonify({"error": "Chunk not found"}), 404)
-            deleted = store.delete_chunk(chunk_id)
-            if not deleted:
-                return make_response(
-                    jsonify({"error": "Failed to delete existing chunk"}), 500
-                )
+
             new_text = text if text is not None else existing_chunk["text"]
-            new_metadata = (
-                metadata if metadata is not None else existing_chunk["metadata"]
-            )
 
-            new_chunk_id = store.add_chunk(new_text, new_metadata)
+            if metadata is not None:
+                new_metadata = existing_chunk["metadata"].copy()
+                new_metadata.update(metadata)
+            else:
+                new_metadata = existing_chunk["metadata"].copy()
 
-            return make_response(
-                jsonify(
-                    {
-                        "message": "Chunk updated successfully",
-                        "new_chunk_id": new_chunk_id,
-                    }
-                ),
-                200,
-            )
+            if text is not None:
+                new_metadata["token_count"] = num_tokens_from_string(new_text)
+
+            try:
+                new_chunk_id = store.add_chunk(new_text, new_metadata)
+
+                deleted = store.delete_chunk(chunk_id)
+                if not deleted:
+                    current_app.logger.warning(f"Failed to delete old chunk {chunk_id}, but new chunk {new_chunk_id} was created")
+
+                return make_response(
+                    jsonify(
+                        {
+                            "message": "Chunk updated successfully",
+                            "chunk_id": new_chunk_id,
+                            "original_chunk_id": chunk_id,
+                        }
+                    ),
+                    200,
+                )
+            except Exception as add_error:
+                current_app.logger.error(f"Failed to add updated chunk: {add_error}")
+                return make_response(
+                    jsonify({"error": "Failed to update chunk - addition failed"}), 500
+                )
         except Exception as e:
             current_app.logger.error(f"Error updating chunk: {e}", exc_info=True)
             return make_response(jsonify({"success": False}), 500)
@@ -3681,3 +3888,51 @@ class ServeImage(Resource):
             return make_response(
                 jsonify({"success": False, "message": "Error retrieving image"}), 500
             )
+
+
+@user_ns.route("/api/directory_structure")
+class DirectoryStructure(Resource):
+    @api.doc(
+        description="Get the directory structure for a document",
+        params={"id": "The document ID"},
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        
+        user = decoded_token.get("sub")
+        doc_id = request.args.get("id")
+        
+        if not doc_id:
+            return make_response(
+                jsonify({"error": "Document ID is required"}), 400
+            )
+            
+        if not ObjectId.is_valid(doc_id):
+            return make_response(jsonify({"error": "Invalid document ID"}), 400)
+            
+        try:
+            doc = sources_collection.find_one({"_id": ObjectId(doc_id), "user": user})
+            if not doc:
+                return make_response(
+                    jsonify({"error": "Document not found or access denied"}), 404
+                )
+                
+            directory_structure = doc.get("directory_structure", {})
+            
+            return make_response(
+                jsonify({
+                    "success": True,
+                    "directory_structure": directory_structure,
+                    "base_path": doc.get("file_path", "")
+                }), 200
+            )
+            
+        except Exception as e:
+            current_app.logger.error(
+                f"Error retrieving directory structure: {e}", exc_info=True
+            )
+            return make_response(
+                jsonify({"success": False, "error": str(e)}), 500
+            )

application/api/user/tasks.py

@@ -11,8 +11,8 @@ from application.worker import (
 
 
 @celery.task(bind=True)
-def ingest(self, directory, formats, job_name, filename, user, dir_name, user_dir):
-    resp = ingest_worker(self, directory, formats, job_name, filename, user, dir_name, user_dir)
+def ingest(self, directory, formats, job_name, user, file_path, filename):
+    resp = ingest_worker(self, directory, formats, job_name, file_path, filename, user)
     return resp
 
 
@@ -22,6 +22,13 @@ def ingest_remote(self, source_data, job_name, user, loader):
     return resp
 
 
+@celery.task(bind=True)
+def reingest_source_task(self, source_id, user):
+    from application.worker import reingest_source_worker
+    resp = reingest_source_worker(self, source_id, user)
+    return resp
+
+
 @celery.task(bind=True)
 def schedule_syncs(self, frequency):
     resp = sync_worker(self, frequency)

application/parser/chunking.py

@@ -32,16 +32,7 @@ class Chunker:
             header, body = "", text  # No header, treat entire text as body
         return header, body
 
-    def combine_documents(self, doc: Document, next_doc: Document) -> Document:
-        combined_text = doc.text + " " + next_doc.text
-        combined_token_count = len(self.encoding.encode(combined_text))
-        new_doc = Document(
-            text=combined_text,
-            doc_id=doc.doc_id,
-            embedding=doc.embedding,
-            extra_info={**(doc.extra_info or {}), "token_count": combined_token_count}
-        )
-        return new_doc
+
     
     def split_document(self, doc: Document) -> List[Document]:
         split_docs = []
@@ -82,26 +73,11 @@ class Chunker:
                 processed_docs.append(doc)
                 i += 1
             elif token_count < self.min_tokens:
-                if i + 1 < len(documents):
-                    next_doc = documents[i + 1]
-                    next_tokens = self.encoding.encode(next_doc.text)
-                    if token_count + len(next_tokens) <= self.max_tokens:
-                        # Combine small documents
-                        combined_doc = self.combine_documents(doc, next_doc)
-                        processed_docs.append(combined_doc)
-                        i += 2
-                    else:
-                        # Keep the small document as is if adding next_doc would exceed max_tokens
-                        doc.extra_info = doc.extra_info or {}
-                        doc.extra_info["token_count"] = token_count
-                        processed_docs.append(doc)
-                        i += 1
-                else:
-                    # No next document to combine with; add the small document as is
-                    doc.extra_info = doc.extra_info or {}
-                    doc.extra_info["token_count"] = token_count
-                    processed_docs.append(doc)
-                    i += 1
+  
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
             else:
                 # Split large documents
                 processed_docs.extend(self.split_document(doc))

application/parser/embedding_pipeline.py

@@ -46,7 +46,7 @@ def embed_and_store_documents(docs, folder_name, source_id, task_status):
         store = VectorCreator.create_vectorstore(
             settings.VECTOR_STORE,
             docs_init=docs_init,
-            source_id=folder_name,
+            source_id=source_id,
             embeddings_key=os.getenv("EMBEDDINGS_KEY"),
         )
     else:

application/parser/file/bulk.py

@@ -15,6 +15,7 @@ from application.parser.file.json_parser import JSONParser
 from application.parser.file.pptx_parser import PPTXParser
 from application.parser.file.image_parser import ImageParser
 from application.parser.schema.base import Document
+from application.utils import num_tokens_from_string
 
 DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
     ".pdf": PDFParser(),
@@ -141,11 +142,12 @@ class SimpleDirectoryReader(BaseReader):
 
         Returns:
             List[Document]: A list of documents.
-
         """
         data: Union[str, List[str]] = ""
         data_list: List[str] = []
         metadata_list = []
+        self.file_token_counts = {}
+        
         for input_file in self.input_files:
             if input_file.suffix in self.file_extractor:
                 parser = self.file_extractor[input_file.suffix]
@@ -156,24 +158,48 @@ class SimpleDirectoryReader(BaseReader):
                 # do standard read
                 with open(input_file, "r", errors=self.errors) as f:
                     data = f.read()
-            # Prepare metadata for this file
-            if self.file_metadata is not None:
-                file_metadata = self.file_metadata(input_file.name)
+            
+            # Calculate token count for this file
+            if isinstance(data, List):
+                file_tokens = sum(num_tokens_from_string(str(d)) for d in data)
             else:
-                # Provide a default empty metadata
-                file_metadata = {'title': '', 'store': ''}
-                # TODO: Find a case with no metadata and check if breaks anything 
+                file_tokens = num_tokens_from_string(str(data))
+            
+            full_path = str(input_file.resolve())
+            self.file_token_counts[full_path] = file_tokens
+            
+            base_metadata = {
+                'title': input_file.name,
+                'token_count': file_tokens,
+            }
+            
+            if hasattr(self, 'input_dir'):
+                try:
+                    relative_path = str(input_file.relative_to(self.input_dir))
+                    base_metadata['source'] = relative_path
+                except ValueError:
+                    base_metadata['source'] = str(input_file)
+            else:
+                base_metadata['source'] = str(input_file)
+
+            if self.file_metadata is not None:
+                custom_metadata = self.file_metadata(input_file.name)
+                base_metadata.update(custom_metadata)
 
             if isinstance(data, List):
                 # Extend data_list with each item in the data list
                 data_list.extend([str(d) for d in data])
-                # For each item in the data list, add the file's metadata to metadata_list
-                metadata_list.extend([file_metadata for _ in data])
+                metadata_list.extend([base_metadata for _ in data])
             else:
-                # Add the single piece of data to data_list
                 data_list.append(str(data))
-                # Add the file's metadata to metadata_list
-                metadata_list.append(file_metadata)
+                metadata_list.append(base_metadata)
+        
+        # Build directory structure if input_dir is provided
+        if hasattr(self, 'input_dir'):
+            self.directory_structure = self.build_directory_structure(self.input_dir)
+            logging.info("Directory structure built successfully")
+        else:
+            self.directory_structure = {}
 
         if concatenate:
             return [Document("\n".join(data_list))]
@@ -181,3 +207,48 @@ class SimpleDirectoryReader(BaseReader):
             return [Document(d, extra_info=m) for d, m in zip(data_list, metadata_list)]
         else:
             return [Document(d) for d in data_list]
+
+    def build_directory_structure(self, base_path):
+        """Build a dictionary representing the directory structure.
+
+        Args:
+            base_path: The base path to start building the structure from.
+
+        Returns:
+            dict: A nested dictionary representing the directory structure.
+        """
+        import mimetypes
+        
+        def build_tree(path):
+            """Helper function to recursively build the directory tree."""
+            result = {}
+            
+            for item in path.iterdir():
+                if self.exclude_hidden and item.name.startswith('.'):
+                    continue
+                    
+                if item.is_dir():
+                    subtree = build_tree(item)
+                    if subtree:
+                        result[item.name] = subtree
+                else:
+                    if self.required_exts is not None and item.suffix not in self.required_exts:
+                        continue
+                    
+                    full_path = str(item.resolve())
+                    file_size_bytes = item.stat().st_size
+                    mime_type = mimetypes.guess_type(item.name)[0] or "application/octet-stream"
+                    
+                    file_info = {
+                        "type": mime_type,
+                        "size_bytes": file_size_bytes
+                    }
+                    
+                    if hasattr(self, 'file_token_counts') and full_path in self.file_token_counts:
+                        file_info["token_count"] = self.file_token_counts[full_path]
+                        
+                    result[item.name] = file_info
+                    
+            return result
+        
+        return build_tree(Path(base_path))

application/storage/base.py

@@ -93,3 +93,32 @@ class BaseStorage(ABC):
             List[str]: List of file paths
         """
         pass
+        
+    @abstractmethod
+    def is_directory(self, path: str) -> bool:
+        """
+        Check if a path is a directory.
+
+        Args:
+            path: Path to check
+
+        Returns:
+            bool: True if the path is a directory
+        """
+        pass
+
+    @abstractmethod
+    def remove_directory(self, directory: str) -> bool:
+        """
+        Remove a directory and all its contents.
+
+        For local storage, this removes the directory and all files/subdirectories within it.
+        For S3 storage, this removes all objects with the directory path as a prefix.
+
+        Args:
+            directory: Directory path to remove
+
+        Returns:
+            bool: True if removal was successful, False otherwise
+        """
+        pass

application/storage/local.py

@@ -101,3 +101,40 @@ class LocalStorage(BaseStorage):
             raise FileNotFoundError(f"File not found: {full_path}")
 
         return processor_func(local_path=full_path, **kwargs)
+
+    def is_directory(self, path: str) -> bool:
+        """
+        Check if a path is a directory in local storage.
+        
+        Args:
+            path: Path to check
+        
+        Returns:
+            bool: True if the path is a directory, False otherwise
+        """
+        full_path = self._get_full_path(path)
+        return os.path.isdir(full_path)
+
+    def remove_directory(self, directory: str) -> bool:
+        """
+        Remove a directory and all its contents from local storage.
+
+        Args:
+            directory: Directory path to remove
+
+        Returns:
+            bool: True if removal was successful, False otherwise
+        """
+        full_path = self._get_full_path(directory)
+
+        if not os.path.exists(full_path):
+            return False
+
+        if not os.path.isdir(full_path):
+            return False
+
+        try:
+            shutil.rmtree(full_path)
+            return True
+        except (OSError, PermissionError):
+            return False

application/storage/s3.py

@@ -130,3 +130,77 @@ class S3Storage(BaseStorage):
             except Exception as e:
                 logging.error(f"Error processing S3 file {path}: {e}", exc_info=True)
                 raise
+
+    def is_directory(self, path: str) -> bool:
+        """
+        Check if a path is a directory in S3 storage.
+
+        In S3, directories are virtual concepts. A path is considered a directory
+        if there are objects with the path as a prefix.
+
+        Args:
+            path: Path to check
+
+        Returns:
+            bool: True if the path is a directory, False otherwise
+        """
+        # Ensure path ends with a slash if not empty
+        if path and not path.endswith('/'):
+            path += '/'
+
+        response = self.s3.list_objects_v2(
+            Bucket=self.bucket_name,
+            Prefix=path,
+            MaxKeys=1
+        )
+
+        return 'Contents' in response
+
+    def remove_directory(self, directory: str) -> bool:
+        """
+        Remove a directory and all its contents from S3 storage.
+
+        In S3, this removes all objects with the directory path as a prefix.
+        Since S3 doesn't have actual directories, this effectively removes
+        all files within the virtual directory structure.
+
+        Args:
+            directory: Directory path to remove
+
+        Returns:
+            bool: True if removal was successful, False otherwise
+        """
+        # Ensure directory ends with a slash if not empty
+        if directory and not directory.endswith('/'):
+            directory += '/'
+
+        try:
+            # Get all objects with the directory prefix
+            objects_to_delete = []
+            paginator = self.s3.get_paginator('list_objects_v2')
+            pages = paginator.paginate(Bucket=self.bucket_name, Prefix=directory)
+
+            for page in pages:
+                if 'Contents' in page:
+                    for obj in page['Contents']:
+                        objects_to_delete.append({'Key': obj['Key']})
+
+            if not objects_to_delete:
+                return False
+
+            batch_size = 1000
+            for i in range(0, len(objects_to_delete), batch_size):
+                batch = objects_to_delete[i:i + batch_size]
+
+                response = self.s3.delete_objects(
+                    Bucket=self.bucket_name,
+                    Delete={'Objects': batch}
+                )
+
+                if 'Errors' in response and response['Errors']:
+                    return False
+
+            return True
+
+        except ClientError:
+            return False

application/vectorstore/faiss.py

@@ -1,5 +1,6 @@
 import os
 import tempfile
+import io
 
 from langchain_community.vectorstores import FAISS
 
@@ -66,8 +67,37 @@ class FaissStore(BaseVectorStore):
     def add_texts(self, *args, **kwargs):
         return self.docsearch.add_texts(*args, **kwargs)
 
-    def save_local(self, *args, **kwargs):
-        return self.docsearch.save_local(*args, **kwargs)
+    def _save_to_storage(self):
+        """
+        Save the FAISS index to storage using temporary directory pattern.
+        Works consistently for both local and S3 storage.
+        """
+        with tempfile.TemporaryDirectory() as temp_dir:
+            self.docsearch.save_local(temp_dir)
+
+            faiss_path = os.path.join(temp_dir, "index.faiss")
+            pkl_path = os.path.join(temp_dir, "index.pkl")
+
+            with open(faiss_path, "rb") as f_faiss:
+                faiss_data = f_faiss.read()
+
+            with open(pkl_path, "rb") as f_pkl:
+                pkl_data = f_pkl.read()
+
+            storage_path = get_vectorstore(self.source_id)
+            self.storage.save_file(io.BytesIO(faiss_data), f"{storage_path}/index.faiss")
+            self.storage.save_file(io.BytesIO(pkl_data), f"{storage_path}/index.pkl")
+
+        return True
+
+    def save_local(self, path=None):
+        if path:
+            os.makedirs(path, exist_ok=True)
+            self.docsearch.save_local(path)
+
+        self._save_to_storage()
+
+        return True
 
     def delete_index(self, *args, **kwargs):
         return self.docsearch.delete(*args, **kwargs)
@@ -103,13 +133,17 @@ class FaissStore(BaseVectorStore):
         return chunks
 
     def add_chunk(self, text, metadata=None):
+        """Add a new chunk and save to storage."""
         metadata = metadata or {}
         doc = Document(text=text, extra_info=metadata).to_langchain_format()
         doc_id = self.docsearch.add_documents([doc])
-        self.save_local(self.path)
+        self._save_to_storage()
         return doc_id
 
+
+
     def delete_chunk(self, chunk_id):
+        """Delete a chunk and save to storage."""
         self.delete_index([chunk_id])
-        self.save_local(self.path)
+        self._save_to_storage()
         return True

application/worker.py

@@ -103,11 +103,23 @@ def download_file(url, params, dest_path):
 
 
 def upload_index(full_path, file_data):
+    files = None
     try:
         if settings.VECTOR_STORE == "faiss":
+            faiss_path = full_path + "/index.faiss"
+            pkl_path = full_path + "/index.pkl"
+
+            if not os.path.exists(faiss_path):
+                logging.error(f"FAISS index file not found: {faiss_path}")
+                raise FileNotFoundError(f"FAISS index file not found: {faiss_path}")
+
+            if not os.path.exists(pkl_path):
+                logging.error(f"FAISS pickle file not found: {pkl_path}")
+                raise FileNotFoundError(f"FAISS pickle file not found: {pkl_path}")
+
             files = {
-                "file_faiss": open(full_path + "/index.faiss", "rb"),
-                "file_pkl": open(full_path + "/index.pkl", "rb"),
+                "file_faiss": open(faiss_path, "rb"),
+                "file_pkl": open(pkl_path, "rb"),
             }
             response = requests.post(
                 urljoin(settings.API_URL, "/api/upload_index"),
@@ -119,11 +131,11 @@ def upload_index(full_path, file_data):
                 urljoin(settings.API_URL, "/api/upload_index"), data=file_data
             )
         response.raise_for_status()
-    except requests.RequestException as e:
+    except (requests.RequestException, FileNotFoundError) as e:
         logging.error(f"Error uploading index: {e}")
         raise
     finally:
-        if settings.VECTOR_STORE == "faiss":
+        if settings.VECTOR_STORE == "faiss" and files is not None:
             for file in files.values():
                 file.close()
 
@@ -200,15 +212,8 @@ def run_agent_logic(agent_config, input_data):
 
 
 def ingest_worker(
-    self,
-    directory,
-    formats,
-    job_name,
-    filename,
-    user,
-    dir_name=None,
-    user_dir=None,
-    retriever="classic",
+    self, directory, formats, job_name, file_path, filename, user, 
+    retriever="classic"
 ):
     """
     Ingest and process documents.
@@ -218,10 +223,9 @@ def ingest_worker(
         directory (str): Specifies the directory for ingesting ('inputs' or 'temp').
         formats (list of str): List of file extensions to consider for ingestion (e.g., [".rst", ".md"]).
         job_name (str): Name of the job for this ingestion task (original, unsanitized).
-        filename (str): Name of the file to be ingested.
+        file_path (str): Complete file path to use consistently throughout the pipeline.
+        filename (str): Original unsanitized filename provided by the user.
         user (str): Identifier for the user initiating the ingestion (original, unsanitized).
-        dir_name (str, optional): Sanitized directory name for filesystem operations.
-        user_dir (str, optional): Sanitized user ID for filesystem operations.
         retriever (str): Type of retriever to use for processing the documents.
 
     Returns:
@@ -234,11 +238,8 @@ def ingest_worker(
     sample = False
 
     storage = StorageCreator.get_storage()
-
-    full_path = os.path.join(directory, user_dir, dir_name)
-    source_file_path = os.path.join(full_path, filename)
-
-    logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": job_name})
+    
+    logging.info(f"Ingest path: {file_path}", extra={"user": user, "job": job_name})
 
     # Create temporary working directory
 
@@ -246,22 +247,46 @@ def ingest_worker(
         try:
             os.makedirs(temp_dir, exist_ok=True)
 
-            # Download file from storage to temp directory
+            if storage.is_directory(file_path):
+                # Handle directory case
+                logging.info(f"Processing directory: {file_path}")
+                files_list = storage.list_files(file_path)
+                
+                for storage_file_path in files_list:
+                    if storage.is_directory(storage_file_path):
+                        continue
+                        
+                    # Create relative path structure in temp directory
+                    rel_path = os.path.relpath(storage_file_path, file_path)
+                    local_file_path = os.path.join(temp_dir, rel_path)
+                    
+                    os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+                    
+                    # Download file
+                    try:
+                        file_data = storage.get_file(storage_file_path)
+                        with open(local_file_path, "wb") as f:
+                            f.write(file_data.read())
+                    except Exception as e:
+                        logging.error(f"Error downloading file {storage_file_path}: {e}")
+                        continue
+            else:
+                # Handle single file case
+                temp_filename = os.path.basename(file_path)
+                temp_file_path = os.path.join(temp_dir, temp_filename)
+                
+                file_data = storage.get_file(file_path)
+                with open(temp_file_path, "wb") as f:
+                    f.write(file_data.read())
 
-            temp_file_path = os.path.join(temp_dir, filename)
-            file_data = storage.get_file(source_file_path)
+                # Handle zip files
+                if temp_filename.endswith(".zip"):
+                    logging.info(f"Extracting zip file: {temp_filename}")
+                    extract_zip_recursive(
+                        temp_file_path, temp_dir, current_depth=0, max_depth=RECURSION_DEPTH
+                    )
 
-            with open(temp_file_path, "wb") as f:
-                f.write(file_data.read())
             self.update_state(state="PROGRESS", meta={"current": 1})
-
-            # Handle zip files
-
-            if filename.endswith(".zip"):
-                logging.info(f"Extracting zip file: {filename}")
-                extract_zip_recursive(
-                    temp_file_path, temp_dir, current_depth=0, max_depth=RECURSION_DEPTH
-                )
             if sample:
                 logging.info(f"Sample mode enabled. Using {limit} documents.")
             reader = SimpleDirectoryReader(
@@ -273,6 +298,9 @@ def ingest_worker(
                 file_metadata=metadata_from_filename,
             )
             raw_docs = reader.load_data()
+            
+            directory_structure = getattr(reader, 'directory_structure', {})
+            logging.info(f"Directory structure from reader: {directory_structure}")
 
             chunker = Chunker(
                 chunking_strategy="classic_chunk",
@@ -299,14 +327,15 @@ def ingest_worker(
                 for i in range(min(5, len(raw_docs))):
                     logging.info(f"Sample document {i}: {raw_docs[i]}")
             file_data = {
-                "name": job_name,  # Use original job_name
+                "name": job_name,
                 "file": filename,
-                "user": user,  # Use original user
+                "user": user,
                 "tokens": tokens,
                 "retriever": retriever,
                 "id": str(id),
                 "type": "local",
-                "original_file_path": source_file_path,
+                "file_path": file_path,
+                "directory_structure": json.dumps(directory_structure),
             }
 
             upload_index(vector_store_path, file_data)
@@ -323,6 +352,252 @@ def ingest_worker(
     }
 
 
+def reingest_source_worker(self, source_id, user):
+    """
+    Re-ingestion worker that handles incremental updates by:
+    1. Adding chunks from newly added files
+    2. Removing chunks from deleted files
+
+    Args:
+        self: Task instance
+        source_id: ID of the source to re-ingest
+        user: User identifier
+
+    Returns:
+        dict: Information about the re-ingestion task
+    """
+    try:
+        from application.vectorstore.vector_creator import VectorCreator
+
+        self.update_state(state="PROGRESS", meta={"current": 10, "status": "Initializing re-ingestion scan"})
+
+        source = sources_collection.find_one({"_id": ObjectId(source_id), "user": user})
+        if not source:
+            raise ValueError(f"Source {source_id} not found or access denied")
+
+        storage = StorageCreator.get_storage()
+        source_file_path = source.get("file_path", "")
+
+        self.update_state(state="PROGRESS", meta={"current": 20, "status": "Scanning current files"})
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Download all files from storage to temp directory, preserving directory structure
+            if storage.is_directory(source_file_path):
+                files_list = storage.list_files(source_file_path)
+
+                for storage_file_path in files_list:
+                    if storage.is_directory(storage_file_path):
+                        continue
+
+
+                    rel_path = os.path.relpath(storage_file_path, source_file_path)
+                    local_file_path = os.path.join(temp_dir, rel_path)
+
+                    os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+                    # Download file
+                    try:
+                        file_data = storage.get_file(storage_file_path)
+                        with open(local_file_path, "wb") as f:
+                            f.write(file_data.read())
+                    except Exception as e:
+                        logging.error(f"Error downloading file {storage_file_path}: {e}")
+                        continue
+
+            reader = SimpleDirectoryReader(
+                input_dir=temp_dir,
+                recursive=True,
+                required_exts=[
+                    ".rst", ".md", ".pdf", ".txt", ".docx", ".csv", ".epub",
+                    ".html", ".mdx", ".json", ".xlsx", ".pptx", ".png",
+                    ".jpg", ".jpeg",
+                ],
+                exclude_hidden=True,
+                file_metadata=metadata_from_filename,
+            )
+            reader.load_data()
+            directory_structure = reader.directory_structure
+            logging.info(f"Directory structure built with token counts: {directory_structure}")
+
+            try:
+                old_directory_structure = source.get("directory_structure") or {}
+                if isinstance(old_directory_structure, str):
+                    try:
+                        old_directory_structure = json.loads(old_directory_structure)
+                    except Exception:
+                        old_directory_structure = {}
+
+                def _flatten_directory_structure(struct, prefix=""):
+                    files = set()
+                    if isinstance(struct, dict):
+                        for name, meta in struct.items():
+                            current_path = os.path.join(prefix, name) if prefix else name
+                            if isinstance(meta, dict) and ("type" in meta and "size_bytes" in meta):
+                                files.add(current_path)
+                            elif isinstance(meta, dict):
+                                files |= _flatten_directory_structure(meta, current_path)
+                    return files
+
+                old_files = _flatten_directory_structure(old_directory_structure)
+                new_files = _flatten_directory_structure(directory_structure)
+
+                added_files = sorted(new_files - old_files)
+                removed_files = sorted(old_files - new_files)
+
+                if added_files:
+                    logging.info(f"Files added since last ingest: {added_files}")
+                else:
+                    logging.info("No files added since last ingest.")
+
+                if removed_files:
+                    logging.info(f"Files removed since last ingest: {removed_files}")
+                else:
+                    logging.info("No files removed since last ingest.")
+
+            except Exception as e:
+                logging.error(f"Error comparing directory structures: {e}", exc_info=True)
+                added_files = []
+                removed_files = []
+            try:
+                if not added_files and not removed_files:
+                    logging.info("No changes detected.")
+                    return {
+                        "source_id": source_id,
+                        "user": user,
+                        "status": "no_changes",
+                        "added_files": [],
+                        "removed_files": [],
+                    }
+
+                vector_store = VectorCreator.create_vectorstore(
+                    settings.VECTOR_STORE,
+                    source_id,
+                    settings.EMBEDDINGS_KEY,
+                )
+
+                self.update_state(state="PROGRESS", meta={"current": 40, "status": "Processing file changes"})
+
+                # 1) Delete chunks from removed files
+                deleted = 0
+                if removed_files:
+                    try:
+                        for ch in vector_store.get_chunks() or []:
+                            metadata = ch.get("metadata", {}) if isinstance(ch, dict) else getattr(ch, "metadata", {})
+                            raw_source = metadata.get("source")
+
+                            source_file = str(raw_source) if raw_source else ""
+
+                            if source_file in removed_files:
+                                cid = ch.get("doc_id")
+                                if cid:
+                                    try:
+                                        vector_store.delete_chunk(cid)
+                                        deleted += 1
+                                    except Exception as de:
+                                        logging.error(f"Failed deleting chunk {cid}: {de}")
+                        logging.info(f"Deleted {deleted} chunks from {len(removed_files)} removed files")
+                    except Exception as e:
+                        logging.error(f"Error during deletion of removed file chunks: {e}", exc_info=True)
+
+                # 2) Add chunks from new files
+                added = 0
+                if added_files:
+                    try:
+                        # Build list of local files for added files only
+                        added_local_files = []
+                        for rel_path in added_files:
+                            local_path = os.path.join(temp_dir, rel_path)
+                            if os.path.isfile(local_path):
+                                added_local_files.append(local_path)
+
+                        if added_local_files:
+                            reader_new = SimpleDirectoryReader(
+                                input_files=added_local_files,
+                                exclude_hidden=True,
+                                errors="ignore",
+                                file_metadata=metadata_from_filename,
+                            )
+                            raw_docs_new = reader_new.load_data()
+                            chunker_new = Chunker(
+                                chunking_strategy="classic_chunk",
+                                max_tokens=MAX_TOKENS,
+                                min_tokens=MIN_TOKENS,
+                                duplicate_headers=False,
+                            )
+                            chunked_new = chunker_new.chunk(documents=raw_docs_new)
+
+                            for file_path, token_count in reader_new.file_token_counts.items():
+                                try:
+                                    rel_path = os.path.relpath(file_path, start=temp_dir)
+                                    path_parts = rel_path.split(os.sep)
+                                    current_dir = directory_structure
+
+                                    for part in path_parts[:-1]:
+                                        if part in current_dir and isinstance(current_dir[part], dict):
+                                            current_dir = current_dir[part]
+                                        else:
+                                            break
+
+                                    filename = path_parts[-1]
+                                    if filename in current_dir and isinstance(current_dir[filename], dict):
+                                        current_dir[filename]["token_count"] = token_count
+                                        logging.info(f"Updated token count for {rel_path}: {token_count}")
+                                except Exception as e:
+                                    logging.warning(f"Could not update token count for {file_path}: {e}")
+
+                            for d in chunked_new:
+                                meta = dict(d.extra_info or {})
+                                try:
+                                    raw_src = meta.get("source")
+                                    if isinstance(raw_src, str) and os.path.isabs(raw_src):
+                                        meta["source"] = os.path.relpath(raw_src, start=temp_dir)
+                                except Exception:
+                                    pass
+
+                                vector_store.add_chunk(d.text, metadata=meta)
+                                added += 1
+                            logging.info(f"Added {added} chunks from {len(added_files)} new files")
+                    except Exception as e:
+                        logging.error(f"Error during ingestion of new files: {e}", exc_info=True)
+
+                # 3) Update source directory structure timestamp
+                try:
+                    total_tokens = sum(reader.file_token_counts.values())
+                    
+                    sources_collection.update_one(
+                        {"_id": ObjectId(source_id)},
+                        {
+                            "$set": {
+                                "directory_structure": directory_structure,
+                                "date": datetime.datetime.now(),
+                                "tokens": total_tokens
+                            }
+                        },
+                    )
+                except Exception as e:
+                    logging.error(f"Error updating directory_structure in DB: {e}", exc_info=True)
+
+                self.update_state(state="PROGRESS", meta={"current": 100, "status": "Re-ingestion completed"})
+
+                return {
+                    "source_id": source_id,
+                    "user": user,
+                    "status": "completed",
+                    "added_files": added_files,
+                    "removed_files": removed_files,
+                    "chunks_added": added,
+                    "chunks_deleted": deleted,
+                }
+            except Exception as e:
+                logging.error(f"Error while processing file changes: {e}", exc_info=True)
+                raise
+
+
+
+    except Exception as e:
+        logging.error(f"Error in reingest_source_worker: {e}", exc_info=True)
+        raise
+
 def remote_worker(
     self,
     source_data,

frontend/src/agents/agentPreviewSlice.ts

@@ -61,7 +61,7 @@ export const fetchPreviewAnswer = createAsyncThunk<
           state.preference.prompt.id,
           state.preference.chunks,
           state.preference.token_limit,
-          (event) => {
+          (event: MessageEvent) => {
             const data = JSON.parse(event.data);
             const targetIndex = indx ?? state.agentPreview.queries.length - 1;
 

frontend/src/api/endpoints.ts

@@ -38,13 +38,24 @@ const endpoints = {
     UPDATE_TOOL_STATUS: '/api/update_tool_status',
     UPDATE_TOOL: '/api/update_tool',
     DELETE_TOOL: '/api/delete_tool',
-    GET_CHUNKS: (docId: string, page: number, per_page: number) =>
-      `/api/get_chunks?id=${docId}&page=${page}&per_page=${per_page}`,
+    GET_CHUNKS: (
+      docId: string,
+      page: number,
+      per_page: number,
+      path?: string,
+      search?: string,
+    ) =>
+      `/api/get_chunks?id=${docId}&page=${page}&per_page=${per_page}${
+        path ? `&path=${encodeURIComponent(path)}` : ''
+      }${search ? `&search=${encodeURIComponent(search)}` : ''}`,
     ADD_CHUNK: '/api/add_chunk',
     DELETE_CHUNK: (docId: string, chunkId: string) =>
       `/api/delete_chunk?id=${docId}&chunk_id=${chunkId}`,
     UPDATE_CHUNK: '/api/update_chunk',
     STORE_ATTACHMENT: '/api/store_attachment',
+    DIRECTORY_STRUCTURE: (docId: string) =>
+      `/api/directory_structure?id=${docId}`,
+    MANAGE_SOURCE_FILES: '/api/manage_source_files',
   },
   CONVERSATION: {
     ANSWER: '/api/answer',

frontend/src/api/services/userService.ts

@@ -86,8 +86,10 @@ const userService = {
     page: number,
     perPage: number,
     token: string | null,
+    path?: string,
+    search?: string,
   ): Promise<any> =>
-    apiClient.get(endpoints.USER.GET_CHUNKS(docId, page, perPage), token),
+    apiClient.get(endpoints.USER.GET_CHUNKS(docId, page, perPage, path, search), token),
   addChunk: (data: any, token: string | null): Promise<any> =>
     apiClient.post(endpoints.USER.ADD_CHUNK, data, token),
   deleteChunk: (
@@ -98,6 +100,10 @@ const userService = {
     apiClient.delete(endpoints.USER.DELETE_CHUNK(docId, chunkId), token),
   updateChunk: (data: any, token: string | null): Promise<any> =>
     apiClient.put(endpoints.USER.UPDATE_CHUNK, data, token),
+  getDirectoryStructure: (docId: string, token: string | null): Promise<any> =>
+    apiClient.get(endpoints.USER.DIRECTORY_STRUCTURE(docId), token),
+  manageSourceFiles: (data: FormData, token: string | null): Promise<any> =>
+    apiClient.postFormData(endpoints.USER.MANAGE_SOURCE_FILES, data, token),
 };
 
 export default userService;

frontend/src/assets/calendar.svg

@@ -0,0 +1,4 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6 7.5C6 7.36739 5.94732 7.24021 5.85355 7.14645C5.75979 7.05268 5.63261 7 5.5 7H4.5C4.36739 7 4.24021 7.05268 4.14645 7.14645C4.05268 7.24021 4 7.36739 4 7.5V8.5C4 8.63261 4.05268 8.75979 4.14645 8.85355C4.24021 8.94732 4.36739 9 4.5 9H5.5C5.63261 9 5.75979 8.94732 5.85355 8.85355C5.94732 8.75979 6 8.63261 6 8.5V7.5ZM6 10.5C6 10.3674 5.94732 10.2402 5.85355 10.1464C5.75979 10.0527 5.63261 10 5.5 10H4.5C4.36739 10 4.24021 10.0527 4.14645 10.1464C4.05268 10.2402 4 10.3674 4 10.5V11.5C4 11.6326 4.05268 11.7598 4.14645 11.8536C4.24021 11.9473 4.36739 12 4.5 12H5.5C5.63261 12 5.75979 11.9473 5.85355 11.8536C5.94732 11.7598 6 11.6326 6 11.5V10.5ZM7.5 7H8.5C8.63261 7 8.75979 7.05268 8.85355 7.14645C8.94732 7.24021 9 7.36739 9 7.5V8.5C9 8.63261 8.94732 8.75979 8.85355 8.85355C8.75979 8.94732 8.63261 9 8.5 9H7.5C7.36739 9 7.24021 8.94732 7.14645 8.85355C7.05268 8.75979 7 8.63261 7 8.5V7.5C7 7.36739 7.05268 7.24021 7.14645 7.14645C7.24021 7.05268 7.36739 7 7.5 7ZM8.5 10H7.5C7.36739 10 7.24021 10.0527 7.14645 10.1464C7.05268 10.2402 7 10.3674 7 10.5V11.5C7 11.6326 7.05268 11.7598 7.14645 11.8536C7.24021 11.9473 7.36739 12 7.5 12H8.5C8.63261 12 8.75979 11.9473 8.85355 11.8536C8.94732 11.7598 9 11.6326 9 11.5V10.5C9 10.3674 8.94732 10.2402 8.85355 10.1464C8.75979 10.0527 8.63261 10 8.5 10ZM10 7.5C10 7.36739 10.0527 7.24021 10.1464 7.14645C10.2402 7.05268 10.3674 7 10.5 7H11.5C11.6326 7 11.7598 7.05268 11.8536 7.14645C11.9473 7.24021 12 7.36739 12 7.5V8.5C12 8.63261 11.9473 8.75979 11.8536 8.85355C11.7598 8.94732 11.6326 9 11.5 9H10.5C10.3674 9 10.2402 8.94732 10.1464 8.85355C10.0527 8.75979 10 8.63261 10 8.5V7.5Z" fill="#848484"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M4.5 0C4.63261 0 4.75979 0.0526784 4.85355 0.146447C4.94732 0.240215 5 0.367392 5 0.5V1H11V0.5C11 0.367392 11.0527 0.240215 11.1464 0.146447C11.2402 0.0526784 11.3674 0 11.5 0C11.6326 0 11.7598 0.0526784 11.8536 0.146447C11.9473 0.240215 12 0.367392 12 0.5V1C13.66 1 15 2.34 15 4V12C15 13.66 13.66 15 12 15H4C2.34 15 1 13.66 1 12V4C1 2.34 2.34 1 4 1V0.5C4 0.367392 4.05268 0.240215 4.14645 0.146447C4.24021 0.0526784 4.36739 0 4.5 0ZM14 4V5H2V4C2 2.9 2.895 2 4 2V2.5C4 2.63261 4.05268 2.75979 4.14645 2.85355C4.24021 2.94732 4.36739 3 4.5 3C4.63261 3 4.75979 2.94732 4.85355 2.85355C4.94732 2.75979 5 2.63261 5 2.5V2H11V2.5C11 2.63261 11.0527 2.75979 11.1464 2.85355C11.2402 2.94732 11.3674 3 11.5 3C11.6326 3 11.7598 2.94732 11.8536 2.85355C11.9473 2.75979 12 2.63261 12 2.5V2C13.1 2 14 2.895 14 4ZM2 12V6H14V12C14 13.1 13.105 14 12 14H4C2.9 14 2 13.105 2 12Z" fill="#848484"/>
+</svg>

frontend/src/assets/disc.svg

@@ -0,0 +1,3 @@
+<svg width="14" height="10" viewBox="0 0 14 10" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M7 0C8.8144 0 10.4902 0.332143 11.739 0.898571C12.362 1.18143 12.9108 1.53643 13.3119 1.96714C13.7172 2.4 14 2.94429 14 3.57143V6.42857C14 7.05571 13.7172 7.59929 13.3119 8.03286C12.9108 8.46357 12.3627 8.81857 11.739 9.10143C10.4902 9.66786 8.8144 10 7 10C5.1856 10 3.5098 9.66786 2.261 9.10143C1.638 8.81857 1.0892 8.46357 0.6881 8.03286C0.2828 7.6 0 7.05571 0 6.42857V3.57143C0 2.94429 0.2828 2.40071 0.6881 1.96714C1.0892 1.53643 1.6373 1.18143 2.261 0.898571C3.5098 0.332143 5.1856 0 7 0ZM12.6 5.77143C12.3375 5.94714 12.047 6.10429 11.739 6.24429C10.4902 6.81071 8.8144 7.14286 7 7.14286C5.1856 7.14286 3.5098 6.81071 2.261 6.24429C1.96243 6.10966 1.67456 5.95157 1.4 5.77143V6.42857C1.4 6.59071 1.47 6.79857 1.7024 7.04857C1.9383 7.30143 2.3128 7.56214 2.8294 7.79643C3.8612 8.26429 5.3354 8.57143 7 8.57143C8.6646 8.57143 10.1388 8.26429 11.1706 7.79643C11.6872 7.56214 12.0617 7.30143 12.2976 7.04857C12.5307 6.79857 12.6 6.59071 12.6 6.42857V5.77143ZM7 1.42857C5.3347 1.42857 3.8612 1.73571 2.8294 2.20357C2.3128 2.43786 1.9383 2.69857 1.7024 2.95143C1.4693 3.20143 1.4 3.40929 1.4 3.57143C1.4 3.73357 1.47 3.94143 1.7024 4.19143C1.9383 4.44429 2.3128 4.705 2.8294 4.93929C3.8612 5.40714 5.3354 5.71429 7 5.71429C8.6646 5.71429 10.1388 5.40714 11.1706 4.93929C11.6872 4.705 12.0617 4.44429 12.2976 4.19143C12.5307 3.94143 12.6 3.73357 12.6 3.57143C12.6 3.40929 12.53 3.20143 12.2976 2.95143C12.0617 2.69857 11.6872 2.43786 11.1706 2.20357C10.1388 1.73643 8.6646 1.42857 7 1.42857Z" fill="#848484"/>
+</svg>

frontend/src/assets/file.svg

@@ -0,0 +1,3 @@
+<svg width="13" height="17" viewBox="0 0 13 17" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0 1.94971C0 0.983707 0.784 0.199707 1.75 0.199707H8.336C8.8 0.199707 9.245 0.383707 9.573 0.712707L12.487 3.62671C12.816 3.95471 13 4.39971 13 4.86371V14.4497C13 14.9138 12.8156 15.359 12.4874 15.6871C12.1592 16.0153 11.7141 16.1997 11.25 16.1997H1.75C1.28587 16.1997 0.840752 16.0153 0.512563 15.6871C0.184375 15.359 0 14.9138 0 14.4497V1.94971ZM1.75 1.69971C1.6837 1.69971 1.62011 1.72605 1.57322 1.77293C1.52634 1.81981 1.5 1.8834 1.5 1.94971V14.4497C1.5 14.5877 1.612 14.6997 1.75 14.6997H11.25C11.3163 14.6997 11.3799 14.6734 11.4268 14.6265C11.4737 14.5796 11.5 14.516 11.5 14.4497V6.19971H8.75C8.28587 6.19971 7.84075 6.01533 7.51256 5.68714C7.18437 5.35896 7 4.91384 7 4.44971V1.69971H1.75ZM8.5 1.76171V4.44971C8.5 4.58771 8.612 4.69971 8.75 4.69971H11.438L11.427 4.68671L8.513 1.77271L8.5 1.76171Z" fill="#59636E"/>
+</svg>

frontend/src/assets/folder.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="15" viewBox="0 0 16 15" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M1.75 0.599915C1.28587 0.599915 0.840752 0.784289 0.512563 1.11248C0.184374 1.44067 0 1.88579 0 2.34991L0 12.8499C0 13.8159 0.784 14.5999 1.75 14.5999H14.25C14.7141 14.5999 15.1592 14.4155 15.4874 14.0874C15.8156 13.7592 16 13.314 16 12.8499V4.34991C16 3.88579 15.8156 3.44067 15.4874 3.11248C15.1592 2.78429 14.7141 2.59991 14.25 2.59991H7.5C7.46119 2.59991 7.42291 2.59088 7.3882 2.57352C7.35348 2.55616 7.32329 2.53096 7.3 2.49991L6.4 1.29991C6.07 0.859915 5.55 0.599915 5 0.599915H1.75Z" fill="#A382E7"/>
+</svg>

frontend/src/assets/outline-source.svg

@@ -0,0 +1,3 @@
+<svg width="21" height="21" viewBox="0 0 21 21" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M19.25 5.25H12.25L10.5 3.5H5.25C4.2875 3.5 3.50875 4.2875 3.50875 5.25L3.5 15.75C3.5 16.7125 4.2875 17.5 5.25 17.5H19.25C20.2125 17.5 21 16.7125 21 15.75V7C21 6.0375 20.2125 5.25 19.25 5.25ZM19.25 15.75H5.25V5.25H9.77375L11.5238 7H19.25V15.75ZM17.5 10.5H7V8.75H17.5V10.5ZM14 14H7V12.25H14V14Z" fill="#949494"/>
+</svg>

frontend/src/assets/search.svg

@@ -0,0 +1,3 @@
+<svg width="15" height="15" viewBox="0 0 15 15" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M10.4798 10.739C9.27414 11.6748 7.7572 12.116 6.23773 11.9728C4.71826 11.8296 3.31047 11.1127 2.30094 9.96806C1.2914 8.82345 0.756002 7.33714 0.803717 5.81168C0.851432 4.28622 1.47868 2.83628 2.55777 1.75699C3.63706 0.677895 5.087 0.0506505 6.61246 0.00293578C8.13792 -0.044779 9.62423 0.490623 10.7688 1.50016C11.9135 2.50969 12.6303 3.91747 12.7736 5.43694C12.9168 6.95641 12.4756 8.47336 11.5398 9.67899L14.5798 12.719C14.6785 12.8107 14.7507 12.9273 14.7887 13.0565C14.8267 13.1858 14.8291 13.3229 14.7958 13.4534C14.7624 13.5839 14.6944 13.703 14.5991 13.7982C14.5037 13.8933 14.3844 13.961 14.2538 13.994C14.1234 14.0274 13.9864 14.0251 13.8573 13.9872C13.7281 13.9494 13.6115 13.8775 13.5198 13.779L10.4798 10.739ZM11.2998 5.99899C11.3087 5.4026 11.1989 4.81039 10.9768 4.25681C10.7547 3.70323 10.4248 3.19934 10.0062 2.77445C9.58757 2.34955 9.08865 2.01214 8.53844 1.78183C7.98824 1.55152 7.39773 1.43292 6.80127 1.43292C6.20481 1.43292 5.6143 1.55152 5.0641 1.78183C4.5139 2.01214 4.01498 2.34955 3.59637 2.77445C3.17777 3.19934 2.84783 3.70323 2.62575 4.25681C2.40367 4.81039 2.29388 5.4026 2.30277 5.99899C2.32039 7.18045 2.80208 8.30756 3.6438 9.13682C4.48552 9.96608 5.61968 10.4309 6.80127 10.4309C7.98286 10.4309 9.11703 9.96608 9.95874 9.13682C10.8005 8.30756 11.2822 7.18045 11.2998 5.99899Z" fill="#59636E"/>
+</svg>

frontend/src/components/Chunks.tsx

@@ -0,0 +1,660 @@
+import React, { useState, useEffect, useRef } from 'react';
+import { useSelector } from 'react-redux';
+import { useTranslation } from 'react-i18next';
+import { selectToken } from '../preferences/preferenceSlice';
+import { useDarkTheme, useLoaderState, useMediaQuery, useOutsideAlerter } from '../hooks';
+import userService from '../api/services/userService';
+import ArrowLeft from '../assets/arrow-left.svg';
+import NoFilesIcon from '../assets/no-files.svg';
+import NoFilesDarkIcon from '../assets/no-files-dark.svg';
+import OutlineSource from '../assets/outline-source.svg';
+import SkeletonLoader from './SkeletonLoader';
+import ConfirmationModal from '../modals/ConfirmationModal';
+import { ActiveState } from '../models/misc';
+import { ChunkType } from '../settings/types';
+import Pagination from './DocumentPagination';
+import FileIcon from '../assets/file.svg';
+import FolderIcon from '../assets/folder.svg';
+import SearchIcon from '../assets/search.svg';
+interface LineNumberedTextareaProps {
+  value: string;
+  onChange: (value: string) => void;
+  placeholder?: string;
+  ariaLabel?: string;
+  className?: string;
+  editable?: boolean;
+  onDoubleClick?: () => void;
+}
+
+const LineNumberedTextarea: React.FC<LineNumberedTextareaProps> = ({
+  value,
+  onChange,
+  placeholder,
+  ariaLabel,
+  className = '',
+  editable = true,
+  onDoubleClick
+}) => {
+  const { isMobile } = useMediaQuery();
+
+  const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
+    onChange(e.target.value);
+  };
+
+  const lineHeight = 19.93;
+  const contentLines = value.split('\n').length;
+
+  const heightOffset = isMobile ? 200 : 300;
+  const minLinesForDisplay = Math.ceil((typeof window !== 'undefined' ? window.innerHeight - heightOffset : 600) / lineHeight);
+  const totalLines = Math.max(contentLines, minLinesForDisplay);
+
+  return (
+    <div className={`relative w-full ${className}`}>
+      <div
+        className="absolute left-0 top-0 w-8 lg:w-12 text-right text-gray-500 dark:text-gray-400 text-xs lg:text-sm font-mono leading-[19.93px] select-none pr-2 lg:pr-3 pointer-events-none"
+        style={{
+          height: `${totalLines * lineHeight}px`
+        }}
+      >
+        {Array.from({ length: totalLines }, (_, i) => (
+          <div
+            key={i + 1}
+            className="flex items-center justify-end h-[19.93px] leading-[19.93px]"
+          >
+            {i + 1}
+          </div>
+        ))}
+      </div>
+      <textarea
+        className={`w-full resize-none bg-transparent dark:text-white font-['Inter'] text-[13.68px] leading-[19.93px] text-[#18181B] outline-none border-none pl-8 lg:pl-12 overflow-hidden ${isMobile ? 'min-h-[calc(100vh-200px)]' : 'min-h-[calc(100vh-300px)]'} ${!editable ? 'select-none' : ''}`}
+        value={value}
+        onChange={editable ? handleChange : undefined}
+        onDoubleClick={onDoubleClick}
+        placeholder={placeholder}
+        aria-label={ariaLabel}
+        rows={totalLines}
+        readOnly={!editable}
+        style={{
+          height: `${totalLines * lineHeight}px`
+        }}
+      />
+    </div>
+  );
+};
+
+interface SearchResult {
+  path: string;
+  isFile: boolean;
+}
+
+interface ChunksProps {
+  documentId: string;
+  documentName?: string;
+  handleGoBack: () => void;
+  path?: string;
+  onFileSearch?: (query: string) => SearchResult[];
+  onFileSelect?: (path: string) => void;
+}
+
+const Chunks: React.FC<ChunksProps> = ({
+  documentId,
+  documentName,
+  handleGoBack,
+  path,
+  onFileSearch,
+  onFileSelect,
+}) => {
+  const [fileSearchQuery, setFileSearchQuery] = useState('');
+  const [fileSearchResults, setFileSearchResults] = useState<SearchResult[]>([]);
+  const searchDropdownRef = useRef<HTMLDivElement>(null);
+  const { t } = useTranslation();
+  const token = useSelector(selectToken);
+  const [isDarkTheme] = useDarkTheme();
+  const [paginatedChunks, setPaginatedChunks] = useState<ChunkType[]>([]);
+  const [page, setPage] = useState(1);
+  const [perPage, setPerPage] = useState(5);
+  const [totalChunks, setTotalChunks] = useState(0);
+  const [loading, setLoading] = useLoaderState(true);
+  const [searchTerm, setSearchTerm] = useState<string>('');
+  const [editingChunk, setEditingChunk] = useState<ChunkType | null>(null);
+  const [editingTitle, setEditingTitle] = useState('');
+  const [editingText, setEditingText] = useState('');
+  const [isAddingChunk, setIsAddingChunk] = useState(false);
+  const [deleteModalState, setDeleteModalState] = useState<ActiveState>('INACTIVE');
+  const [chunkToDelete, setChunkToDelete] = useState<ChunkType | null>(null);
+  const [isEditing, setIsEditing] = useState(false);
+
+  const pathParts = path ? path.split('/') : [];
+
+  const fetchChunks = () => {
+    setLoading(true);
+    try {
+      userService
+        .getDocumentChunks(documentId, page, perPage, token, path, searchTerm)
+        .then((response) => {
+          if (!response.ok) {
+            setLoading(false);
+            setPaginatedChunks([]);
+            throw new Error('Failed to fetch chunks data');
+          }
+          return response.json();
+        })
+        .then((data) => {
+          setPage(data.page);
+          setPerPage(data.per_page);
+          setTotalChunks(data.total);
+          setPaginatedChunks(data.chunks);
+          setLoading(false);
+        })
+        .catch((error) => {
+          setLoading(false);
+          setPaginatedChunks([]);
+        });
+    } catch (e) {
+      setLoading(false);
+      setPaginatedChunks([]);
+    }
+  };
+
+  const handleAddChunk = (title: string, text: string) => {
+    if (!text.trim()) {
+      return;
+    }
+
+    try {
+      const metadata = {
+        source: path || documentName,
+        source_id: documentId,
+        title: title,
+      };
+
+      userService
+        .addChunk(
+          {
+            id: documentId,
+            text: text,
+            metadata: metadata,
+          },
+          token,
+        )
+        .then((response) => {
+          if (!response.ok) {
+            throw new Error('Failed to add chunk');
+          }
+          fetchChunks();
+        });
+    } catch (e) {
+      console.log(e);
+    }
+  };
+
+  const handleUpdateChunk = (title: string, text: string, chunk: ChunkType) => {
+
+    if (!text.trim()) {
+      return;
+    }
+
+    const originalTitle = chunk.metadata?.title || '';
+    const originalText = chunk.text || '';
+
+    if (title === originalTitle && text === originalText) {
+      return;
+    }
+
+    try {
+      userService
+        .updateChunk(
+          {
+            id: documentId,
+            chunk_id: chunk.doc_id,
+            text: text,
+            metadata: {
+              title: title,
+            },
+          },
+          token,
+        )
+        .then((response) => {
+          if (!response.ok) {
+            throw new Error('Failed to update chunk');
+          }
+          fetchChunks();
+        });
+    } catch (e) {
+      console.log(e);
+    }
+  };
+
+  const handleDeleteChunk = (chunk: ChunkType) => {
+    try {
+      userService
+        .deleteChunk(documentId, chunk.doc_id, token)
+        .then((response) => {
+          if (!response.ok) {
+            throw new Error('Failed to delete chunk');
+          }
+          setEditingChunk(null);
+          fetchChunks();
+        });
+    } catch (e) {
+      console.log(e);
+    }
+  };
+
+  const confirmDeleteChunk = (chunk: ChunkType) => {
+    setChunkToDelete(chunk);
+    setDeleteModalState('ACTIVE');
+  };
+
+  const handleConfirmedDelete = () => {
+    if (chunkToDelete) {
+      handleDeleteChunk(chunkToDelete);
+      setDeleteModalState('INACTIVE');
+      setChunkToDelete(null);
+    }
+  };
+
+  const handleCancelDelete = () => {
+    setDeleteModalState('INACTIVE');
+    setChunkToDelete(null);
+  };
+
+  useEffect(() => {
+    const delayDebounceFn = setTimeout(() => {
+      if (page !== 1) {
+        setPage(1);
+      } else {
+        fetchChunks();
+      }
+    }, 300);
+
+    return () => clearTimeout(delayDebounceFn);
+  }, [searchTerm]);
+
+  useEffect(() => {
+    !loading && fetchChunks();
+  }, [page, perPage, path]);
+  
+  useEffect(() => {
+    setSearchTerm('');
+    setPage(1);
+  }, [path]);
+
+  const filteredChunks = paginatedChunks;
+
+  const renderPathNavigation = () => {
+    return (
+      <div className="mb-0 min-h-[38px] flex flex-col sm:flex-row sm:items-center sm:justify-between text-base gap-2">
+        <div className="flex w-full items-center sm:w-auto">
+          <button
+            className="mr-3 flex h-[29px] w-[29px] items-center justify-center rounded-full border p-2 text-sm text-gray-400 dark:border-0 dark:bg-[#28292D] dark:text-gray-500 dark:hover:bg-[#2E2F34] transition-all duration-200 font-medium"
+            onClick={editingChunk ? () => setEditingChunk(null) : isAddingChunk ? () => setIsAddingChunk(false) : handleGoBack}
+          >
+            <img src={ArrowLeft} alt="left-arrow" className="h-3 w-3" />
+          </button>
+
+          <div className="flex items-center flex-wrap">
+            {/* Removed the directory icon */}
+            <span className="text-[#7D54D1] font-semibold break-words">
+              {documentName}
+            </span>
+
+            {pathParts.length > 0 && (
+              <>
+                <span className="mx-1 text-gray-500 flex-shrink-0">/</span>
+                {pathParts.map((part, index) => (
+                  <React.Fragment key={index}>
+                    <span className={`break-words ${
+                      index < pathParts.length - 1 
+                        ? 'text-[#7D54D1] font-medium' 
+                        : 'text-gray-700 dark:text-gray-300'
+                    }`}>
+                      {part}
+                    </span>
+                    {index < pathParts.length - 1 && (
+                      <span className="mx-1 text-gray-500 flex-shrink-0">/</span>
+                    )}
+                  </React.Fragment>
+                ))}
+              </>
+            )}
+          </div>
+        </div>
+
+        <div className="flex flex-row flex-nowrap items-center gap-2 w-full sm:w-auto justify-end mt-2 sm:mt-0 overflow-x-auto">
+          {editingChunk ? (
+            !isEditing ? (
+              <>
+                <button
+                  className="bg-purple-30 hover:bg-violets-are-blue flex h-[38px] min-w-[108px] items-center justify-center rounded-full px-4 text-[14px] whitespace-nowrap text-white font-medium"
+                  onClick={() => setIsEditing(true)}
+                >
+                  {t('modals.chunk.edit')}
+                </button>
+                <button
+                  className="rounded-full border border-solid border-red-500 px-4 py-1 text-[14px] text-nowrap text-red-500 hover:bg-red-500 hover:text-white h-[38px] min-w-[108px] flex items-center justify-center font-medium"
+                  onClick={() => {
+                    confirmDeleteChunk(editingChunk);
+                  }}
+                >
+                  {t('modals.chunk.delete')}
+                </button>
+              </>
+            ) : (
+              <>
+                <button
+                  onClick={() => {
+                    setIsEditing(false);
+                  }}
+                  className="dark:text-light-gray cursor-pointer rounded-full px-4 py-1 text-sm font-medium hover:bg-gray-100 dark:bg-transparent dark:hover:bg-[#767183]/50 text-nowrap h-[38px] min-w-[108px] flex items-center justify-center"
+                >
+                  {t('modals.chunk.cancel')}
+                </button>
+                <button
+                  onClick={() => {
+                    if (editingText.trim()) {
+                      const hasChanges = editingTitle !== (editingChunk?.metadata?.title || '') ||
+                                       editingText !== (editingChunk?.text || '');
+
+                      if (hasChanges) {
+                        handleUpdateChunk(editingTitle, editingText, editingChunk);
+                      }
+                      setIsEditing(false);
+                      setEditingChunk(null);
+                    }
+                  }}
+                  disabled={!editingText.trim() || (editingTitle === (editingChunk?.metadata?.title || '') && editingText === (editingChunk?.text || ''))}
+                  className={`text-nowrap rounded-full px-4 py-1 text-[14px] text-white transition-all flex items-center justify-center h-[38px] min-w-[108px] font-medium ${
+                    editingText.trim() && (editingTitle !== (editingChunk?.metadata?.title || '') || editingText !== (editingChunk?.text || ''))
+                      ? 'bg-purple-30 hover:bg-violets-are-blue cursor-pointer'
+                      : 'bg-gray-400 cursor-not-allowed'
+                  }`}
+                >
+                  {t('modals.chunk.save')}
+                </button>
+              </>
+            )
+          ) : isAddingChunk ? (
+            <>
+              <button
+                onClick={() => setIsAddingChunk(false)}
+                className="dark:text-light-gray cursor-pointer rounded-full px-4 py-1 text-sm font-medium hover:bg-gray-100 dark:bg-transparent dark:hover:bg-[#767183]/50 text-nowrap h-[38px] min-w-[108px] flex items-center justify-center"
+              >
+                {t('modals.chunk.cancel')}
+              </button>
+              <button
+                onClick={() => {
+                  if (editingText.trim()) {
+                    handleAddChunk(editingTitle, editingText);
+                    setIsAddingChunk(false);
+                  }
+                }}
+                disabled={!editingText.trim()}
+                className={`text-nowrap rounded-full px-4 py-1 text-[14px] text-white transition-all flex items-center justify-center h-[38px] min-w-[108px] font-medium ${
+                  editingText.trim()
+                    ? 'bg-purple-30 hover:bg-violets-are-blue cursor-pointer'
+                    : 'bg-gray-400 cursor-not-allowed'
+                }`}
+              >
+                {t('modals.chunk.add')}
+              </button>
+            </>
+          ) : null}
+        </div>
+      </div>
+    );
+  };
+
+  // File search handling
+  const handleFileSearchChange = (query: string) => {
+    setFileSearchQuery(query);
+    if (query.trim() && onFileSearch) {
+      const results = onFileSearch(query);
+      setFileSearchResults(results);
+    } else {
+      setFileSearchResults([]);
+    }
+  };
+
+  const handleSearchResultClick = (result: SearchResult) => {
+    if (!onFileSelect) return;
+
+    if (result.isFile) {
+      onFileSelect(result.path);
+    } else {
+      // For directories, navigate to the directory and return to file tree
+      onFileSelect(result.path);
+      handleGoBack();
+    }
+    setFileSearchQuery('');
+    setFileSearchResults([]);
+  };
+
+  useOutsideAlerter(
+    searchDropdownRef,
+    () => {
+      setFileSearchQuery('');
+      setFileSearchResults([]);
+    },
+    [], // No additional dependencies
+    false // Don't handle escape key
+  );
+
+  const renderFileSearch = () => {
+    return (
+      <div className="relative" ref={searchDropdownRef}>
+        <div className="relative flex items-center">
+          <div className="absolute left-3 pointer-events-none">
+            <img src={SearchIcon} alt="Search" className="w-4 h-4" />
+          </div>
+          <input
+            type="text"
+            value={fileSearchQuery}
+            onChange={(e) => handleFileSearchChange(e.target.value)}
+            placeholder={t('settings.sources.searchFiles')}
+            className={`w-full h-[38px] border border-[#D1D9E0] pl-10 pr-4 py-2 dark:border-[#6A6A6A] 
+              ${fileSearchQuery 
+                ? 'rounded-t-[6px]' 
+                : 'rounded-[6px]'
+              } 
+              bg-transparent focus:outline-none dark:text-[#E0E0E0] transition-all duration-200`}
+          />
+        </div>
+
+        {fileSearchQuery && (
+          <div className="absolute z-10 max-h-[calc(100vh-200px)] w-full overflow-hidden rounded-b-[6px] border border-t-0 border-[#D1D9E0] bg-white shadow-lg dark:border-[#6A6A6A] dark:bg-[#1F2023]">
+            <div className="max-h-[calc(100vh-200px)] overflow-y-auto overflow-x-hidden">
+              {fileSearchResults.length === 0 ? (
+                <div className="py-2 text-center text-sm text-gray-500 dark:text-gray-400">
+                  {t('settings.sources.noResults')}
+                </div>
+              ) : (
+                fileSearchResults.map((result, index) => (
+                  <div
+                    key={index}
+                    title={result.path}
+                    onClick={() => handleSearchResultClick(result)}
+                    className={`flex cursor-pointer items-center px-3 py-2 hover:bg-[#ECEEEF] dark:hover:bg-[#27282D] ${
+                      index !== fileSearchResults.length - 1
+                        ? 'border-b border-[#D1D9E0] dark:border-[#6A6A6A]'
+                        : ''
+                    }`}
+                  >
+                    <img
+                      src={result.isFile ? FileIcon : FolderIcon}
+                      alt={result.isFile ? 'File' : 'Folder'}
+                      className="mr-2 h-4 w-4 flex-shrink-0"
+                    />
+                    <span className="text-sm dark:text-[#E0E0E0] truncate">
+                      {result.path.split('/').pop() || result.path}
+                    </span>
+                  </div>
+                ))
+              )}
+            </div>
+          </div>
+        )}
+      </div>
+    );
+  };
+
+  return (
+    <div className="flex flex-col">
+      <div className="mb-2">
+        {renderPathNavigation()}
+      </div>
+      <div className="flex gap-4">
+        {onFileSearch && onFileSelect && (
+          <div className="hidden lg:block w-[198px]">
+            {renderFileSearch()}
+          </div>
+        )}
+
+        {/* Right side: Chunks content */}
+        <div className="flex-1">
+          {!editingChunk && !isAddingChunk ? (
+            <>
+              <div className="mb-3 flex flex-col sm:flex-row items-start sm:items-center justify-between gap-3">
+                <div className="flex-1 w-full flex items-center border border-[#D1D9E0] dark:border-[#6A6A6A] rounded-md overflow-hidden h-[38px]">
+                  <div className="px-4 flex items-center text-gray-700 dark:text-[#E0E0E0] font-medium whitespace-nowrap h-full">
+                    {totalChunks > 999999
+                      ? `${(totalChunks / 1000000).toFixed(2)}M`
+                      : totalChunks > 999
+                        ? `${(totalChunks / 1000).toFixed(2)}K`
+                        : totalChunks} {t('settings.sources.chunks')}
+                  </div>
+                  <div className="h-full w-[1px] bg-[#D1D9E0] dark:bg-[#6A6A6A]"></div>
+                  <div className="flex-1 h-full">
+                    <input
+                      type="text"
+                      placeholder={t('settings.sources.searchPlaceholder')}
+                      value={searchTerm}
+                      onChange={(e) => setSearchTerm(e.target.value)}
+                      className="w-full h-full px-3 py-2 bg-transparent border-none outline-none font-normal text-[13.56px] leading-[100%] dark:text-[#E0E0E0]"
+                    />
+                  </div>
+                </div>
+                <button
+                  className="bg-purple-30 hover:bg-violets-are-blue flex h-[38px] w-full sm:w-auto min-w-[108px] items-center justify-center rounded-full px-4 text-[14px] whitespace-normal text-white shrink-0 font-medium"
+                  title={t('settings.sources.addChunk')}
+                  onClick={() => {
+                    setIsAddingChunk(true);
+                    setEditingTitle('');
+                    setEditingText('');
+                  }}
+                >
+                  {t('settings.sources.addChunk')}
+                </button>
+              </div>
+              {loading ? (
+                <div className="w-full grid grid-cols-1 sm:[grid-template-columns:repeat(auto-fit,minmax(400px,1fr))] gap-4 justify-items-start">
+                  <SkeletonLoader component="chunkCards" count={perPage} />
+                </div>
+              ) : (
+                <div className="w-full grid grid-cols-1 sm:[grid-template-columns:repeat(auto-fit,minmax(400px,1fr))] gap-4 justify-items-start">
+                  {filteredChunks.length === 0 ? (
+                    <div className="col-span-full w-full min-h-[50vh] flex flex-col items-center justify-center text-center text-gray-500 dark:text-gray-400">
+                      <img
+                        src={isDarkTheme ? NoFilesDarkIcon : NoFilesIcon}
+                        alt={t('settings.sources.noChunksAlt')}
+                        className="mx-auto mb-2 h-24 w-24"
+                      />
+                      {t('settings.sources.noChunks')}
+                    </div>
+                  ) : (
+                    filteredChunks.map((chunk, index) => (
+                      <div
+                        key={index}
+                        className="transform transition-transform duration-200 hover:scale-105 relative flex h-[197px] flex-col justify-between rounded-[5.86px] border border-[#D1D9E0] dark:border-[#6A6A6A] overflow-hidden cursor-pointer w-full max-w-[487px]"
+                        onClick={() => {
+                          setEditingChunk(chunk);
+                          setEditingTitle(chunk.metadata?.title || '');
+                          setEditingText(chunk.text || '');
+                        }}
+                      >
+                        <div className="w-full">
+                          <div className="flex w-full items-center justify-between border-b border-[#D1D9E0] bg-[#F6F8FA] dark:bg-[#27282D] dark:border-[#6A6A6A] px-4 py-3">
+                            <div className="text-[#59636E] text-sm dark:text-[#E0E0E0]">
+                              {chunk.metadata.token_count ? chunk.metadata.token_count.toLocaleString() : '-'} {t('settings.sources.tokensUnit')}
+                            </div>
+                          </div>
+                          <div className="px-4 pt-3 pb-6">
+                            <p className="font-['Inter'] text-[13.68px] leading-[19.93px] text-[#18181B] dark:text-[#E0E0E0] line-clamp-6 font-normal">
+                              {chunk.text}
+                            </p>
+                          </div>
+                        </div>
+                      </div>
+                    ))
+                  )}
+                </div>
+              )}
+            </>
+          ) : isAddingChunk ? (
+            <div className="w-full">
+              <div className="relative border border-[#D1D9E0] dark:border-[#6A6A6A] rounded-lg overflow-hidden">
+                <LineNumberedTextarea
+                  value={editingText}
+                  onChange={setEditingText}
+                  ariaLabel={t('modals.chunk.promptText')}
+                  editable={true}
+                />
+              </div>
+            </div>
+          ) : editingChunk && (
+            <div className="w-full">
+              <div className="relative flex flex-col rounded-[5.86px] border border-[#D1D9E0] dark:border-[#6A6A6A] overflow-hidden w-full">
+                <div className="flex w-full items-center justify-between border-b border-[#D1D9E0] bg-[#F6F8FA] dark:bg-[#27282D] dark:border-[#6A6A6A] px-4 py-3">
+                  <div className="text-[#59636E] text-sm dark:text-[#E0E0E0]">
+                    {editingChunk.metadata.token_count ? editingChunk.metadata.token_count.toLocaleString() : '-'} {t('settings.sources.tokensUnit')}
+                  </div>
+                </div>
+                <div className="p-4 overflow-hidden">
+                  <LineNumberedTextarea
+                    value={isEditing ? editingText : editingChunk.text}
+                    onChange={setEditingText}
+                    ariaLabel={t('modals.chunk.promptText')}
+                    editable={isEditing}
+                    onDoubleClick={() => {
+                      if (!isEditing) {
+                        setIsEditing(true);
+                        setEditingTitle(editingChunk.metadata.title || '');
+                        setEditingText(editingChunk.text);
+                      }
+                    }}
+                  />
+                </div>
+              </div>
+            </div>
+          )}
+
+          {!loading && totalChunks > perPage && !editingChunk && !isAddingChunk && (
+            <Pagination
+              currentPage={page}
+              totalPages={Math.ceil(totalChunks / perPage)}
+              rowsPerPage={perPage}
+              onPageChange={setPage}
+              onRowsPerPageChange={(rows) => {
+                setPerPage(rows);
+                setPage(1);
+              }}
+            />
+          )}
+        </div>
+      </div>
+
+      {/* Delete Confirmation Modal */}
+      <ConfirmationModal
+        message={t('modals.chunk.deleteConfirmation')}
+        modalState={deleteModalState}
+        setModalState={setDeleteModalState}
+        handleSubmit={handleConfirmedDelete}
+        handleCancel={handleCancelDelete}
+        submitLabel={t('modals.chunk.delete')}
+        variant="danger"
+      />
+    </div>
+  );
+};
+
+export default Chunks;

frontend/src/components/ContextMenu.tsx

@@ -82,14 +82,14 @@ export default function ContextMenu({
     // Adjust position based on specified position
     switch (position) {
       case 'bottom-left':
-        left = rect.left + scrollX - offset.x;
+        left = rect.right + scrollX - menuWidth + offset.x;
         break;
       case 'top-right':
         top = rect.top + scrollY - offset.y - menuHeight;
         break;
       case 'top-left':
         top = rect.top + scrollY - offset.y - menuHeight;
-        left = rect.left + scrollX - offset.x;
+        left = rect.right + scrollX - menuWidth + offset.x;
         break;
       // bottom-right is default
     }

frontend/src/components/FileTreeComponent.tsx

@@ -0,0 +1,887 @@
+import React, { useState, useRef, useEffect } from 'react';
+import { useTranslation } from 'react-i18next';
+import { useSelector } from 'react-redux';
+import { selectToken } from '../preferences/preferenceSlice';
+import Chunks from './Chunks';
+import ContextMenu, { MenuOption } from './ContextMenu';
+import userService from '../api/services/userService';
+import FileIcon from '../assets/file.svg';
+import FolderIcon from '../assets/folder.svg';
+import ArrowLeft from '../assets/arrow-left.svg';
+import ThreeDots from '../assets/three-dots.svg';
+import EyeView from '../assets/eye-view.svg';
+import OutlineSource from '../assets/outline-source.svg';
+import Trash from '../assets/red-trash.svg';
+import SearchIcon from '../assets/search.svg';
+import { useOutsideAlerter } from '../hooks';
+import ConfirmationModal from '../modals/ConfirmationModal';
+
+interface FileNode {
+  type?: string;
+  token_count?: number;
+  size_bytes?: number;
+  [key: string]: any;
+}
+
+interface DirectoryStructure {
+  [key: string]: FileNode;
+}
+
+interface FileTreeComponentProps {
+  docId: string;
+  sourceName: string;
+  onBackToDocuments: () => void;
+}
+
+interface SearchResult {
+  name: string;
+  path: string;
+  isFile: boolean;
+}
+
+const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
+  docId,
+  sourceName,
+  onBackToDocuments,
+}) => {
+  const { t } = useTranslation();
+  const [loading, setLoading] = useState<boolean>(true);
+  const [error, setError] = useState<string | null>(null);
+  const [directoryStructure, setDirectoryStructure] =
+    useState<DirectoryStructure | null>(null);
+  const [currentPath, setCurrentPath] = useState<string[]>([]);
+  const token = useSelector(selectToken);
+  const [activeMenuId, setActiveMenuId] = useState<string | null>(null);
+  const menuRefs = useRef<{
+    [key: string]: React.RefObject<HTMLDivElement | null>;
+  }>({});
+  const [selectedFile, setSelectedFile] = useState<{
+    id: string;
+    name: string;
+  } | null>(null);
+  const [searchQuery, setSearchQuery] = useState('');
+  const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
+  const searchDropdownRef = useRef<HTMLDivElement>(null);
+  const currentOpRef = useRef<null | 'add' | 'remove' | 'remove_directory'>(
+    null,
+  );
+
+  const [deleteModalState, setDeleteModalState] = useState<
+    'ACTIVE' | 'INACTIVE'
+  >('INACTIVE');
+  const [itemToDelete, setItemToDelete] = useState<{
+    name: string;
+    isFile: boolean;
+  } | null>(null);
+
+  type QueuedOperation = {
+    operation: 'add' | 'remove' | 'remove_directory';
+    files?: File[];
+    filePath?: string;
+    directoryPath?: string;
+    parentDirPath?: string;
+  };
+  const opQueueRef = useRef<QueuedOperation[]>([]);
+  const processingRef = useRef(false);
+  const [queueLength, setQueueLength] = useState(0);
+
+  useOutsideAlerter(
+    searchDropdownRef,
+    () => {
+      setSearchQuery('');
+      setSearchResults([]);
+    },
+    [],
+    false,
+  );
+
+  const handleFileClick = (fileName: string) => {
+    const fullPath = [...currentPath, fileName].join('/');
+    setSelectedFile({
+      id: fullPath,
+      name: fileName,
+    });
+  };
+
+  useEffect(() => {
+    const fetchDirectoryStructure = async () => {
+      try {
+        setLoading(true);
+        const response = await userService.getDirectoryStructure(docId, token);
+        const data = await response.json();
+
+        if (data && data.directory_structure) {
+          setDirectoryStructure(data.directory_structure);
+        } else {
+          setError('Invalid response format');
+        }
+      } catch (err) {
+        setError('Failed to load directory structure');
+        console.error(err);
+      } finally {
+        setLoading(false);
+      }
+    };
+
+    if (docId) {
+      fetchDirectoryStructure();
+    }
+  }, [docId, token]);
+
+  const formatBytes = (bytes: number): string => {
+    if (bytes === 0) return '0 Bytes';
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+  };
+
+  const navigateToDirectory = (dirName: string) => {
+    setCurrentPath((prev) => [...prev, dirName]);
+  };
+
+  const navigateUp = () => {
+    setCurrentPath((prev) => prev.slice(0, -1));
+  };
+
+  const getCurrentDirectory = (): DirectoryStructure => {
+    if (!directoryStructure) return {};
+
+    let structure = directoryStructure;
+    if (typeof structure === 'string') {
+      try {
+        structure = JSON.parse(structure);
+      } catch (e) {
+        console.error(
+          'Error parsing directory structure in getCurrentDirectory:',
+          e,
+        );
+        return {};
+      }
+    }
+
+    if (typeof structure !== 'object' || structure === null) {
+      return {};
+    }
+
+    let current: any = structure;
+    for (const dir of currentPath) {
+      if (
+        current[dir] &&
+        typeof current[dir] === 'object' &&
+        !current[dir].type
+      ) {
+        current = current[dir];
+      } else {
+        return {};
+      }
+    }
+    return current;
+  };
+
+  const handleBackNavigation = () => {
+    if (selectedFile) {
+      setSelectedFile(null);
+    } else if (currentPath.length === 0) {
+      if (onBackToDocuments) {
+        onBackToDocuments();
+      }
+    } else {
+      navigateUp();
+    }
+  };
+
+  const getMenuRef = (itemId: string) => {
+    if (!menuRefs.current[itemId]) {
+      menuRefs.current[itemId] = React.createRef<HTMLDivElement>();
+    }
+    return menuRefs.current[itemId];
+  };
+
+  const handleMenuClick = (e: React.MouseEvent, itemId: string) => {
+    e.preventDefault();
+    e.stopPropagation();
+
+    if (activeMenuId === itemId) {
+      setActiveMenuId(null);
+      return;
+    }
+    setActiveMenuId(itemId);
+  };
+
+  const getActionOptions = (
+    name: string,
+    isFile: boolean,
+    _itemId: string,
+  ): MenuOption[] => {
+    const options: MenuOption[] = [];
+
+    options.push({
+      icon: EyeView,
+      label: t('settings.sources.view'),
+      onClick: (event: React.SyntheticEvent) => {
+        event.stopPropagation();
+        if (isFile) {
+          handleFileClick(name);
+        } else {
+          navigateToDirectory(name);
+        }
+      },
+      iconWidth: 18,
+      iconHeight: 18,
+      variant: 'primary',
+    });
+
+    options.push({
+      icon: Trash,
+      label: t('convTile.delete'),
+      onClick: (event: React.SyntheticEvent) => {
+        event.stopPropagation();
+        confirmDeleteItem(name, isFile);
+      },
+      iconWidth: 18,
+      iconHeight: 18,
+      variant: 'danger',
+    });
+
+    return options;
+  };
+
+  const confirmDeleteItem = (name: string, isFile: boolean) => {
+    setItemToDelete({ name, isFile });
+    setDeleteModalState('ACTIVE');
+    setActiveMenuId(null);
+  };
+
+  const handleConfirmedDelete = async () => {
+    if (itemToDelete) {
+      await handleDeleteFile(itemToDelete.name, itemToDelete.isFile);
+      setDeleteModalState('INACTIVE');
+      setItemToDelete(null);
+    }
+  };
+
+  const handleCancelDelete = () => {
+    setDeleteModalState('INACTIVE');
+    setItemToDelete(null);
+  };
+
+  const manageSource = async (
+    operation: 'add' | 'remove' | 'remove_directory',
+    files?: File[] | null,
+    filePath?: string,
+    directoryPath?: string,
+    parentDirPath?: string,
+  ) => {
+    currentOpRef.current = operation;
+
+    try {
+      const formData = new FormData();
+      formData.append('source_id', docId);
+      formData.append('operation', operation);
+
+      if (operation === 'add' && files && files.length) {
+        formData.append('parent_dir', parentDirPath ?? currentPath.join('/'));
+
+        for (let i = 0; i < files.length; i++) {
+          formData.append('file', files[i]);
+        }
+      } else if (operation === 'remove' && filePath) {
+        const filePaths = JSON.stringify([filePath]);
+        formData.append('file_paths', filePaths);
+      } else if (operation === 'remove_directory' && directoryPath) {
+        formData.append('directory_path', directoryPath);
+      }
+
+      const response = await userService.manageSourceFiles(formData, token);
+      const result = await response.json();
+
+      if (result.success && result.reingest_task_id) {
+        if (operation === 'add') {
+          console.log('Files uploaded successfully:', result.added_files);
+        } else if (operation === 'remove') {
+          console.log('Files deleted successfully:', result.removed_files);
+        } else if (operation === 'remove_directory') {
+          console.log(
+            'Directory deleted successfully:',
+            result.removed_directory,
+          );
+        }
+        console.log('Reingest task started:', result.reingest_task_id);
+
+        const maxAttempts = 30;
+        const pollInterval = 2000;
+
+        for (let attempt = 0; attempt < maxAttempts; attempt++) {
+          try {
+            const statusResponse = await userService.getTaskStatus(
+              result.reingest_task_id,
+              token,
+            );
+            const statusData = await statusResponse.json();
+
+            console.log(
+              `Task status (attempt ${attempt + 1}):`,
+              statusData.status,
+            );
+
+            if (statusData.status === 'SUCCESS') {
+              console.log('Task completed successfully');
+
+              const structureResponse = await userService.getDirectoryStructure(
+                docId,
+                token,
+              );
+              const structureData = await structureResponse.json();
+
+              if (structureData && structureData.directory_structure) {
+                setDirectoryStructure(structureData.directory_structure);
+                currentOpRef.current = null;
+                return true;
+              }
+              break;
+            } else if (statusData.status === 'FAILURE') {
+              console.error('Task failed');
+              break;
+            }
+
+            await new Promise((resolve) => setTimeout(resolve, pollInterval));
+          } catch (error) {
+            console.error('Error polling task status:', error);
+            break;
+          }
+        }
+      } else {
+        throw new Error(
+          `Failed to ${operation} ${operation === 'remove_directory' ? 'directory' : 'file(s)'}`,
+        );
+      }
+    } catch (error) {
+      const actionText =
+        operation === 'add'
+          ? 'uploading'
+          : operation === 'remove_directory'
+            ? 'deleting directory'
+            : 'deleting file(s)';
+      const errorText =
+        operation === 'add'
+          ? 'upload'
+          : operation === 'remove_directory'
+            ? 'delete directory'
+            : 'delete file(s)';
+      console.error(`Error ${actionText}:`, error);
+      setError(`Failed to ${errorText}`);
+    } finally {
+      currentOpRef.current = null;
+    }
+
+    return false;
+  };
+
+  const processQueue = async () => {
+    if (processingRef.current) return;
+    processingRef.current = true;
+    try {
+      while (opQueueRef.current.length > 0) {
+        const nextOp = opQueueRef.current.shift()!;
+        setQueueLength(opQueueRef.current.length);
+        await manageSource(
+          nextOp.operation,
+          nextOp.files,
+          nextOp.filePath,
+          nextOp.directoryPath,
+          nextOp.parentDirPath,
+        );
+      }
+    } finally {
+      processingRef.current = false;
+    }
+  };
+
+  const enqueueOperation = (op: QueuedOperation) => {
+    opQueueRef.current.push(op);
+    setQueueLength(opQueueRef.current.length);
+    if (!processingRef.current) {
+      void processQueue();
+    }
+  };
+
+  const handleAddFile = () => {
+    const fileInput = document.createElement('input');
+    fileInput.type = 'file';
+    fileInput.multiple = true;
+    fileInput.accept =
+      '.rst,.md,.pdf,.txt,.docx,.csv,.epub,.html,.mdx,.json,.xlsx,.pptx,.png,.jpg,.jpeg';
+
+    fileInput.onchange = async (event) => {
+      const fileList = (event.target as HTMLInputElement).files;
+      if (!fileList || fileList.length === 0) return;
+      const files = Array.from(fileList);
+      enqueueOperation({
+        operation: 'add',
+        files,
+        parentDirPath: currentPath.join('/'),
+      });
+    };
+
+    fileInput.click();
+  };
+
+  const handleDeleteFile = async (name: string, isFile: boolean) => {
+    // Construct the full path to the file or directory
+    const itemPath = [...currentPath, name].join('/');
+
+    if (isFile) {
+      enqueueOperation({ operation: 'remove', filePath: itemPath });
+    } else {
+      enqueueOperation({
+        operation: 'remove_directory',
+        directoryPath: itemPath,
+      });
+    }
+  };
+
+  const renderPathNavigation = () => {
+    return (
+      <div className="mb-0 min-h-[38px] flex flex-col gap-2 text-base sm:flex-row sm:items-center sm:justify-between">
+        {/* Left side with path navigation */}
+        <div className="flex w-full items-center sm:w-auto">
+          <button
+            className="mr-3 flex h-[29px] w-[29px] items-center justify-center rounded-full border p-2 text-sm text-gray-400 dark:border-0 dark:bg-[#28292D] dark:text-gray-500 dark:hover:bg-[#2E2F34] font-medium"
+            onClick={handleBackNavigation}
+          >
+            <img src={ArrowLeft} alt="left-arrow" className="h-3 w-3" />
+          </button>
+
+          <div className="flex flex-wrap items-center">
+            <span className="text-[#7D54D1] font-semibold break-words">
+              {sourceName}
+            </span>
+            {currentPath.length > 0 && (
+              <>
+                <span className="mx-1 flex-shrink-0 text-gray-500">/</span>
+                {currentPath.map((dir, index) => (
+                  <React.Fragment key={index}>
+                    <span className="break-words text-gray-700 dark:text-gray-300">
+                      {dir}
+                    </span>
+                    {index < currentPath.length - 1 && (
+                      <span className="mx-1 flex-shrink-0 text-gray-500">
+                        /
+                      </span>
+                    )}
+                  </React.Fragment>
+                ))}
+              </>
+            )}
+            {selectedFile && (
+              <>
+                <span className="mx-1 flex-shrink-0 text-gray-500">/</span>
+                <span className="break-words text-gray-700 dark:text-gray-300">
+                  {selectedFile.name}
+                </span>
+              </>
+            )}
+          </div>
+        </div>
+
+        <div className="flex relative flex-row flex-nowrap items-center gap-2 w-full sm:w-auto justify-end mt-2 sm:mt-0">
+
+          {processingRef.current && (
+            <div className="text-sm text-gray-500">
+              {currentOpRef.current === 'add'
+                ? t('settings.sources.uploadingFilesTitle')
+                : t('settings.sources.deletingTitle')}
+            </div>
+          )}
+
+            {renderFileSearch()}
+
+          {/* Add file button */}
+          {!processingRef.current && (
+            <button
+              onClick={handleAddFile}
+              className="bg-purple-30 hover:bg-violets-are-blue flex h-[38px] min-w-[108px] items-center justify-center rounded-full px-4 text-[14px] whitespace-nowrap text-white font-medium"
+              title={t('settings.sources.addFile')}
+            >
+              {t('settings.sources.addFile')}
+            </button>
+          )}
+        </div>
+      </div>
+    );
+  };
+  const calculateDirectoryStats = (
+    structure: DirectoryStructure,
+  ): { totalSize: number; totalTokens: number } => {
+    let totalSize = 0;
+    let totalTokens = 0;
+
+    Object.entries(structure).forEach(([_, node]) => {
+      if (node.type) {
+        // It's a file
+        totalSize += node.size_bytes || 0;
+        totalTokens += node.token_count || 0;
+      } else {
+        // It's a directory, recurse
+        const stats = calculateDirectoryStats(node);
+        totalSize += stats.totalSize;
+        totalTokens += stats.totalTokens;
+      }
+    });
+
+    return { totalSize, totalTokens };
+  };
+
+  const renderFileTree = (structure: DirectoryStructure): React.ReactNode[] => {
+    // Separate directories and files
+    const entries = Object.entries(structure);
+    const directories = entries.filter(([_, node]) => !node.type);
+    const files = entries.filter(([_, node]) => node.type);
+
+    // Create parent directory row
+    const parentRow =
+      currentPath.length > 0
+        ? [
+          <tr
+            key="parent-dir"
+            className="cursor-pointer border-b border-[#D1D9E0] hover:bg-[#ECEEEF] dark:border-[#6A6A6A] dark:hover:bg-[#27282D]"
+            onClick={navigateUp}
+          >
+            <td className="px-2 py-2 lg:px-4">
+              <div className="flex items-center">
+                <img
+                  src={FolderIcon}
+                  alt={t('settings.sources.parentFolderAlt')}
+                  className="mr-2 h-4 w-4 flex-shrink-0"
+                />
+                <span className="truncate text-sm dark:text-[#E0E0E0]">
+                  ..
+                </span>
+              </div>
+            </td>
+            <td className="px-2 py-2 text-sm lg:px-4 dark:text-[#E0E0E0]">
+              -
+            </td>
+            <td className="px-2 py-2 text-sm lg:px-4 dark:text-[#E0E0E0]">
+              -
+            </td>
+            <td className="w-10 px-2 py-2 text-sm lg:px-4"></td>
+          </tr>,
+        ]
+        : [];
+
+    // Render directories first, then files
+    return [
+      ...parentRow,
+      ...directories.map(([name, node]) => {
+        const itemId = `dir-${name}`;
+        const menuRef = getMenuRef(itemId);
+        const dirStats = calculateDirectoryStats(node as DirectoryStructure);
+
+        return (
+          <tr
+            key={itemId}
+            className="cursor-pointer border-b border-[#D1D9E0] hover:bg-[#ECEEEF] dark:border-[#6A6A6A] dark:hover:bg-[#27282D]"
+            onClick={() => navigateToDirectory(name)}
+          >
+            <td className="px-2 py-2 lg:px-4">
+              <div className="flex min-w-0 items-center">
+                <img
+                  src={FolderIcon}
+                  alt={t('settings.sources.folderAlt')}
+                  className="mr-2 h-4 w-4 flex-shrink-0"
+                />
+                <span className="truncate text-sm dark:text-[#E0E0E0]">
+                  {name}
+                </span>
+              </div>
+            </td>
+            <td className="px-2 py-2 text-sm lg:px-4 dark:text-[#E0E0E0]">
+              {dirStats.totalTokens > 0
+                ? dirStats.totalTokens.toLocaleString()
+                : '-'}
+            </td>
+            <td className="px-2 py-2 text-sm lg:px-4 dark:text-[#E0E0E0]">
+              {dirStats.totalSize > 0 ? formatBytes(dirStats.totalSize) : '-'}
+            </td>
+            <td className="w-10 px-2 py-2 text-sm lg:px-4">
+              <div ref={menuRef} className="relative">
+                <button
+                  onClick={(e) => handleMenuClick(e, itemId)}
+                  className="inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E] font-medium"
+                  aria-label={t('settings.sources.menuAlt')}
+                >
+                  <img
+                    src={ThreeDots}
+                    alt={t('settings.sources.menuAlt')}
+                    className="opacity-60 hover:opacity-100"
+                  />
+                </button>
+                <ContextMenu
+                  isOpen={activeMenuId === itemId}
+                  setIsOpen={(isOpen) =>
+                    setActiveMenuId(isOpen ? itemId : null)
+                  }
+                  options={getActionOptions(name, false, itemId)}
+                  anchorRef={menuRef}
+                  position="bottom-left"
+                  offset={{ x: -4, y: 4 }}
+                />
+              </div>
+            </td>
+          </tr>
+        );
+      }),
+      ...files.map(([name, node]) => {
+        const itemId = `file-${name}`;
+        const menuRef = getMenuRef(itemId);
+
+        return (
+          <tr
+            key={itemId}
+            className="cursor-pointer border-b border-[#D1D9E0] hover:bg-[#ECEEEF] dark:border-[#6A6A6A] dark:hover:bg-[#27282D]"
+            onClick={() => handleFileClick(name)}
+          >
+            <td className="px-2 py-2 lg:px-4">
+              <div className="flex min-w-0 items-center">
+                <img
+                  src={FileIcon}
+                  alt={t('settings.sources.fileAlt')}
+                  className="mr-2 h-4 w-4 flex-shrink-0"
+                />
+                <span className="truncate text-sm dark:text-[#E0E0E0]">
+                  {name}
+                </span>
+              </div>
+            </td>
+            <td className="px-2 py-2 text-sm lg:px-4 dark:text-[#E0E0E0]">
+              {node.token_count?.toLocaleString() || '-'}
+            </td>
+            <td className="px-2 py-2 text-sm md:px-4 dark:text-[#E0E0E0]">
+              {node.size_bytes ? formatBytes(node.size_bytes) : '-'}
+            </td>
+            <td className="w-10 px-2 py-2 text-sm lg:px-4">
+              <div ref={menuRef} className="relative">
+                <button
+                  onClick={(e) => handleMenuClick(e, itemId)}
+                  className="inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E] font-medium"
+                  aria-label={t('settings.sources.menuAlt')}
+                >
+                  <img
+                    src={ThreeDots}
+                    alt={t('settings.sources.menuAlt')}
+                    className="opacity-60 hover:opacity-100"
+                  />
+                </button>
+                <ContextMenu
+                  isOpen={activeMenuId === itemId}
+                  setIsOpen={(isOpen) =>
+                    setActiveMenuId(isOpen ? itemId : null)
+                  }
+                  options={getActionOptions(name, true, itemId)}
+                  anchorRef={menuRef}
+                  position="bottom-left"
+                  offset={{ x: -4, y: 4 }}
+                />
+              </div>
+            </td>
+          </tr>
+        );
+      }),
+    ];
+  };
+  const currentDirectory = getCurrentDirectory();
+
+  const searchFiles = (
+    query: string,
+    structure: DirectoryStructure,
+    currentPath: string[] = [],
+  ): SearchResult[] => {
+    let results: SearchResult[] = [];
+
+    Object.entries(structure).forEach(([name, node]) => {
+      const fullPath = [...currentPath, name].join('/');
+
+      if (name.toLowerCase().includes(query.toLowerCase())) {
+        results.push({
+          name,
+          path: fullPath,
+          isFile: !!node.type,
+        });
+      }
+
+      if (!node.type) {
+        // If it's a directory, search recursively
+        results = [
+          ...results,
+          ...searchFiles(query, node as DirectoryStructure, [
+            ...currentPath,
+            name,
+          ]),
+        ];
+      }
+    });
+
+    return results;
+  };
+
+  const handleSearchSelect = (result: SearchResult) => {
+    if (result.isFile) {
+      const pathParts = result.path.split('/');
+      const fileName = pathParts.pop() || '';
+      setCurrentPath(pathParts);
+
+      setSelectedFile({
+        id: result.path,
+        name: fileName,
+      });
+    } else {
+      setCurrentPath(result.path.split('/'));
+      setSelectedFile(null);
+    }
+    setSearchQuery('');
+    setSearchResults([]);
+  };
+
+  const renderFileSearch = () => {
+    return (
+      <div className="relative w-52" ref={searchDropdownRef}>
+        <input
+          type="text"
+          value={searchQuery}
+          onChange={(e) => {
+            setSearchQuery(e.target.value);
+            if (directoryStructure) {
+              setSearchResults(searchFiles(e.target.value, directoryStructure));
+            }
+          }}
+          placeholder={t('settings.sources.searchFiles')}
+          className={`w-full h-[38px] border border-[#D1D9E0] px-4 py-2 dark:border-[#6A6A6A] 
+              ${searchQuery ? 'rounded-t-[24px]' : 'rounded-[24px]'} 
+              bg-transparent focus:outline-none dark:text-[#E0E0E0]`}
+        />
+
+        {searchQuery && (
+          <div className="absolute top-full left-0 right-0 z-10 max-h-[calc(100vh-200px)] w-full overflow-hidden rounded-b-[12px] border border-t-0 border-[#D1D9E0] bg-white shadow-lg dark:border-[#6A6A6A] dark:bg-[#1F2023] transition-all duration-200">
+            <div className="max-h-[calc(100vh-200px)] overflow-y-auto overflow-x-hidden overscroll-contain">
+              {searchResults.length === 0 ? (
+                <div className="py-2 text-center text-sm text-gray-500 dark:text-gray-400">
+                  {t('settings.sources.noResults')}
+                </div>
+              ) : (
+                searchResults.map((result, index) => (
+                  <div
+                    key={index}
+                    onClick={() => handleSearchSelect(result)}
+                    title={result.path}
+                    className={`flex min-w-0 cursor-pointer items-center px-3 py-2 hover:bg-[#ECEEEF] dark:hover:bg-[#27282D] ${index !== searchResults.length - 1
+                        ? 'border-b border-[#D1D9E0] dark:border-[#6A6A6A]'
+                        : ''
+                      }`}
+                  >
+                    <img
+                      src={result.isFile ? FileIcon : FolderIcon}
+                      alt={
+                        result.isFile
+                          ? t('settings.sources.fileAlt')
+                          : t('settings.sources.folderAlt')
+                      }
+                      className="mr-2 h-4 w-4 flex-shrink-0"
+                    />
+                    <span className="text-sm dark:text-[#E0E0E0] truncate flex-1">
+                      {result.path.split('/').pop() || result.path}
+                    </span>
+                  </div>
+                ))
+              )}
+            </div>
+          </div>
+        )}
+      </div>
+    );
+  };
+
+  const handleFileSearch = (searchQuery: string) => {
+    if (directoryStructure) {
+      return searchFiles(searchQuery, directoryStructure);
+    }
+    return [];
+  };
+
+  const handleFileSelect = (path: string) => {
+    const pathParts = path.split('/');
+    const fileName = pathParts.pop() || '';
+    setCurrentPath(pathParts);
+    setSelectedFile({
+      id: path,
+      name: fileName,
+    });
+  };
+
+  return (
+    <div>
+      {selectedFile ? (
+        <div className="flex">
+          <div className="flex-1">
+            <Chunks
+              documentId={docId}
+              documentName={sourceName}
+              handleGoBack={() => setSelectedFile(null)}
+              path={selectedFile.id}
+              onFileSearch={handleFileSearch}
+              onFileSelect={handleFileSelect}
+            />
+          </div>
+        </div>
+      ) : (
+        <div className="flex w-full max-w-full flex-col overflow-hidden">
+          <div className="mb-2">{renderPathNavigation()}</div>
+
+          <div className="w-full">
+            <div className="overflow-x-auto rounded-[6px] border border-[#D1D9E0] dark:border-[#6A6A6A]">
+              <table className="w-full min-w-[600px] table-auto bg-transparent">
+                <thead className="bg-gray-100 dark:bg-[#27282D]">
+                  <tr className="border-b border-[#D1D9E0] dark:border-[#6A6A6A]">
+                    <th className="min-w-[200px] px-2 py-3 text-left text-sm font-medium text-gray-700 lg:px-4 dark:text-[#59636E]">
+                      {t('settings.sources.fileName')}
+                    </th>
+                    <th className="min-w-[80px] px-2 py-3 text-left text-sm font-medium text-gray-700 lg:px-4 dark:text-[#59636E]">
+                      {t('settings.sources.tokens')}
+                    </th>
+                    <th className="min-w-[80px] px-2 py-3 text-left text-sm font-medium text-gray-700 lg:px-4 dark:text-[#59636E]">
+                      {t('settings.sources.size')}
+                    </th>
+                    <th className="w-[60px] px-2 py-3 text-left text-sm font-medium text-gray-700 lg:px-4 dark:text-[#59636E]">
+                      <span className="sr-only">
+                        {t('settings.sources.actions')}
+                      </span>
+                    </th>
+                  </tr>
+                </thead>
+                <tbody className="[&>tr:last-child]:border-b-0">
+                  {renderFileTree(currentDirectory)}
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      )}
+      <ConfirmationModal
+        message={
+          itemToDelete?.isFile
+            ? t('settings.sources.confirmDelete')
+            : t('settings.sources.deleteDirectoryWarning', { name: itemToDelete?.name })
+        }
+        modalState={deleteModalState}
+        setModalState={setDeleteModalState}
+        handleSubmit={handleConfirmedDelete}
+        handleCancel={handleCancelDelete}
+        submitLabel={t('convTile.delete')}
+        variant="danger"
+      />
+    </div>
+  );
+};
+
+export default FileTreeComponent;

frontend/src/components/Head.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 
-interface DocumentHeadProps {
+interface HeadProps {
   title?: string;
   description?: string;
   keywords?: string;
@@ -13,7 +13,7 @@ interface DocumentHeadProps {
   children?: React.ReactNode;
 }
 
-export function DocumentHead({
+export function Head({
   title,
   description,
   keywords,
@@ -24,7 +24,7 @@ export function DocumentHead({
   twitterTitle,
   twitterDescription,
   children,
-}: DocumentHeadProps) {
+}: HeadProps) {
   return (
     <>
       {title && <title>{title}</title>}

frontend/src/components/SettingsBar.tsx

@@ -10,7 +10,7 @@ const useTabs = () => {
   const { t } = useTranslation();
   const tabs = [
     t('settings.general.label'),
-    t('settings.documents.label'),
+    t('settings.sources.label'),
     t('settings.analytics.label'),
     t('settings.logs.label'),
     t('settings.tools.label'),

frontend/src/components/SkeletonLoader.tsx

@@ -8,7 +8,9 @@ interface SkeletonLoaderProps {
     | 'logs'
     | 'table'
     | 'chatbot'
-    | 'dropdown';
+    | 'dropdown'
+    | 'chunkCards'
+    | 'sourceCards';
 }
 
 const SkeletonLoader: React.FC<SkeletonLoaderProps> = ({
@@ -182,6 +184,62 @@ const SkeletonLoader: React.FC<SkeletonLoaderProps> = ({
     </>
   );
 
+  const renderChunkCards = () => (
+    <>
+      {Array.from({ length: count }).map((_, index) => (
+        <div
+          key={`chunk-skel-${index}`}
+          className="relative flex h-[197px] flex-col rounded-[5.86px] border border-[#D1D9E0] dark:border-[#6A6A6A] overflow-hidden w-full max-w-[487px] animate-pulse"
+        >
+          <div className="w-full">
+            <div className="flex w-full items-center justify-between border-b border-[#D1D9E0] bg-[#F6F8FA] dark:bg-[#27282D] dark:border-[#6A6A6A] px-4 py-3">
+              <div className="h-4 bg-gray-300 dark:bg-gray-600 rounded w-20"></div>
+            </div>
+            <div className="px-4 pt-4 pb-6 space-y-3">
+              <div className="h-3 bg-gray-200 dark:bg-gray-700 rounded w-full"></div>
+              <div className="h-3 bg-gray-200 dark:bg-gray-700 rounded w-11/12"></div>
+              <div className="h-3 bg-gray-200 dark:bg-gray-700 rounded w-5/6"></div>
+              <div className="h-3 bg-gray-200 dark:bg-gray-700 rounded w-4/5"></div>
+              <div className="h-3 bg-gray-200 dark:bg-gray-700 rounded w-3/4"></div>
+              <div className="h-3 bg-gray-200 dark:bg-gray-700 rounded w-2/3"></div>
+            </div>
+          </div>
+        </div>
+      ))}
+    </>
+  );
+
+  const renderSourceCards = () => (
+    <>
+      {Array.from({ length: count }).map((_, idx) => (
+        <div
+          key={`source-skel-${idx}`}
+          className="flex h-[130px] w-full flex-col rounded-2xl bg-[#F9F9F9] dark:bg-[#383838] p-3 animate-pulse"
+        >
+          <div className="w-full flex-1">
+            <div className="flex w-full items-center justify-between gap-2">
+              <div className="flex-1">
+                <div className="h-[13px] w-full rounded bg-gray-200 dark:bg-gray-700"></div>
+              </div>
+              <div className="w-6 h-6 rounded bg-gray-200 dark:bg-gray-700"></div>
+            </div>
+          </div>
+
+          <div className="flex flex-col items-start justify-start gap-1 pt-3">
+            <div className="flex items-center gap-2 mb-1">
+              <div className="w-3 h-3 rounded bg-gray-200 dark:bg-gray-700"></div>
+              <div className="h-[12px] w-20 rounded bg-gray-200 dark:bg-gray-700"></div>
+            </div>
+            <div className="flex items-center gap-2">
+              <div className="w-3 h-3 rounded bg-gray-200 dark:bg-gray-700"></div>
+              <div className="h-[12px] w-16 rounded bg-gray-200 dark:bg-gray-700"></div>
+            </div>
+          </div>
+        </div>
+      ))}
+    </>
+  );
+
   const componentMap = {
     table: renderTable,
     chatbot: renderChatbot,
@@ -189,8 +247,11 @@ const SkeletonLoader: React.FC<SkeletonLoaderProps> = ({
     logs: renderLogs,
     default: renderDefault,
     analysis: renderAnalysis,
+    chunkCards: renderChunkCards,
+    sourceCards: renderSourceCards,
   };
 
+
   const render = componentMap[component] || componentMap.default;
 
   return <>{render()}</>;

frontend/src/components/SourceDropdown.tsx

@@ -158,7 +158,7 @@ function SourceDropdown({
         </div>
       )}
       <ConfirmationModal
-        message={t('settings.documents.deleteWarning', {
+        message={t('settings.sources.deleteWarning', {
           name: documentToDelete?.name,
         })}
         modalState={deleteModalState}

frontend/src/components/SourcesPopup.tsx

@@ -135,7 +135,7 @@ export default function SourcesPopup({
             type="text"
             value={searchTerm}
             onChange={(e) => setSearchTerm(e.target.value)}
-            placeholder={t('settings.documents.searchPlaceholder')}
+            placeholder={t('settings.sources.searchPlaceholder')}
             borderVariant="thin"
             className="mb-4"
             labelBgClassName="bg-lotion dark:bg-charleston-green-2"
@@ -203,11 +203,11 @@ export default function SourcesPopup({
 
         <div className="shrink-0 px-4 py-4 opacity-75 transition-opacity duration-200 hover:opacity-100 md:px-6">
           <a
-            href="/settings/documents"
+            href="/settings/sources"
             className="text-violets-are-blue inline-flex items-center gap-2 text-base font-medium"
             onClick={onClose}
           >
-            {t('settings.documents.goToDocuments')}
+            {t('settings.sources.goToSources')}
             <img src={RedirectIcon} alt="Redirect" className="h-3 w-3" />
           </a>
         </div>
@@ -217,7 +217,7 @@ export default function SourcesPopup({
             onClick={handleUploadClick}
             className="border-violets-are-blue text-violets-are-blue hover:bg-violets-are-blue w-auto rounded-full border px-4 py-2 text-[14px] font-medium transition-colors duration-200 hover:text-white"
           >
-            {t('settings.documents.uploadNew')}
+            {t('settings.sources.uploadNew')}
           </button>
         </div>
       </div>

frontend/src/conversation/SharedConversation.tsx

@@ -23,7 +23,7 @@ import {
   updateQuery,
 } from './sharedConversationSlice';
 import { selectCompletedAttachments } from '../upload/uploadSlice';
-import { DocumentHead } from '../components/DocumentHead';
+import { Head as DocumentHead } from '../components/Head';
 
 export const SharedConversation = () => {
   const navigate = useNavigate();

frontend/src/locale/en.json

@@ -52,16 +52,18 @@
       "default": "Default",
       "add": "Add"
     },
-    "documents": {
-      "title": "This table contains all the documents that are available to you and those you have uploaded",
-      "label": "Documents",
-      "name": "Document Name",
+    "sources": {
+      "title": "Here you can manage all of the source file that are available to you and those you have uploaded.",
+      "label": "Sources",
+      "name": "Source Name",
       "date": "Vector Date",
       "type": "Type",
       "tokenUsage": "Token Usage",
-      "noData": "No existing Documents",
+      "noData": "No existing Sources",
       "searchPlaceholder": "Search...",
       "addNew": "Add New",
+      "addSource": "Add Source",
+      "addChunk": "Add Chunk",
       "preLoaded": "Pre-loaded",
       "private": "Private",
       "sync": "Sync",
@@ -74,12 +76,32 @@
       "actions": "Actions",
       "view": "View",
       "deleteWarning": "Are you sure you want to delete \"{{name}}\"?",
-      "backToAll": "Back to all documents",
+      "confirmDelete": "Are you sure you want to delete this file? This action cannot be undone.",
+      "backToAll": "Back to all sources",
       "chunks": "Chunks",
       "noChunks": "No chunks found",
       "noChunksAlt": "No chunks found",
-      "goToDocuments": "Go to Documents",
-      "uploadNew": "Upload new"
+      "goToSources": "Go to Sources",
+      "uploadNew": "Upload new",
+      "searchFiles": "Search files...",
+      "noResults": "No results found",
+      "fileName": "Name",
+      "tokens": "Tokens",
+      "size": "Size",
+      "fileAlt": "File",
+      "folderAlt": "Folder",
+      "parentFolderAlt": "Parent folder",
+      "menuAlt": "Menu",
+      "tokensUnit": "tokens",
+      "editAlt": "Edit",
+      "uploading": "Uploading…",
+      "deleting": "Deleting…",
+      "queued": "Queued: {{count}}",
+      "addFile": "Add file",
+      "uploadingFilesTitle": "Uploading files...",
+      "deletingTitle": "Deleting...",
+      "deleteDirectoryWarning": "Are you sure you want to delete the directory \"{{name}}\" and all its contents? This action cannot be undone.",
+      "searchAlt": "Search"
     },
     "apiKeys": {
       "label": "Chatbots",
@@ -250,13 +272,14 @@
     },
     "chunk": {
       "add": "Add Chunk",
-      "edit": "Edit Chunk",
+      "edit": "Edit",
       "title": "Title",
       "enterTitle": "Enter title",
       "bodyText": "Body text",
       "promptText": "Prompt Text",
-      "update": "Update",
+      "save": "Save",
       "close": "Close",
+      "cancel": "Cancel",
       "delete": "Delete",
       "deleteConfirmation": "Are you sure you want to delete this chunk?"
     }

frontend/src/locale/es.json

@@ -52,16 +52,18 @@
       "default": "Predeterminado",
       "addNew": "Añadir Nuevo"
     },
-    "documents": {
-      "title": "Esta tabla contiene todos los documentos que están disponibles para ti y los que has subido",
-      "label": "Documentos",
-      "name": "Nombre del Documento",
+    "sources": {
+      "title": "Aquí puedes gestionar todos los archivos fuente que están disponibles para ti y los que has subido.",
+      "label": "Fuentes",
+      "name": "Nombre de la Fuente",
       "date": "Fecha de Vector",
       "type": "Tipo",
       "tokenUsage": "Uso de Tokens",
-      "noData": "No hay documentos existentes",
+      "noData": "No hay fuentes existentes",
       "searchPlaceholder": "Buscar...",
       "addNew": "Agregar Nuevo",
+      "addSource": "Agregar Fuente",
+      "addChunk": "Agregar Fragmento",
       "preLoaded": "Precargado",
       "private": "Privado",
       "sync": "Sincronizar",
@@ -74,12 +76,32 @@
       "actions": "Acciones",
       "view": "Ver",
       "deleteWarning": "¿Estás seguro de que deseas eliminar \"{{name}}\"?",
-      "backToAll": "Volver a todos los documentos",
+      "confirmDelete": "¿Estás seguro de que deseas eliminar este archivo? Esta acción no se puede deshacer.",
+      "backToAll": "Volver a todas las fuentes",
       "chunks": "Fragmentos",
       "noChunks": "No se encontraron fragmentos",
       "noChunksAlt": "No se encontraron fragmentos",
-      "goToDocuments": "Ir a Documentos",
-      "uploadNew": "Subir nuevo"
+      "goToSources": "Ir a Fuentes",
+      "uploadNew": "Subir nuevo",
+      "searchFiles": "Buscar archivos...",
+      "noResults": "No se encontraron resultados",
+      "fileName": "Nombre",
+      "tokens": "Tokens",
+      "size": "Tamaño",
+      "fileAlt": "Archivo",
+      "folderAlt": "Carpeta",
+      "parentFolderAlt": "Carpeta padre",
+      "menuAlt": "Menú",
+      "tokensUnit": "tokens",
+      "editAlt": "Editar",
+      "uploading": "Subiendo…",
+      "deleting": "Eliminando…",
+      "queued": "En cola: {{count}}",
+      "addFile": "Añadir archivo",
+      "uploadingFilesTitle": "Subiendo archivos...",
+      "deletingTitle": "Eliminando...",
+      "deleteDirectoryWarning": "¿Está seguro de que desea eliminar el directorio \"{{name}}\" y todo su contenido? Esta acción no se puede deshacer.",
+      "searchAlt": "Buscar"
     },
     "apiKeys": {
       "label": "Chatbots",
@@ -250,13 +272,14 @@
     },
     "chunk": {
       "add": "Agregar Fragmento",
-      "edit": "Editar Fragmento",
+      "edit": "Editar",
       "title": "Título",
       "enterTitle": "Ingresar título",
       "bodyText": "Texto del cuerpo",
       "promptText": "Texto del prompt",
-      "update": "Actualizar",
+      "save": "Guardar",
       "close": "Cerrar",
+      "cancel": "Cancelar",
       "delete": "Eliminar",
       "deleteConfirmation": "¿Estás seguro de que deseas eliminar este fragmento?"
     }

frontend/src/locale/jp.json

@@ -52,16 +52,18 @@
       "default": "デフォルト",
       "add": "追加"
     },
-    "documents": {
-      "title": "この表には、利用可能なすべてのドキュメントとアップロードしたドキュメントが含まれています",
-      "label": "ドキュメント",
-      "name": "ドキュメント名",
+    "sources": {
+      "title": "ここでは、利用可能なすべてのソースファイルとアップロードしたファイルを管理できます。",
+      "label": "ソース",
+      "name": "ソース名",
       "date": "ベクトル日付",
       "type": "タイプ",
       "tokenUsage": "トークン使用量",
-      "noData": "既存のドキュメントがありません",
+      "noData": "既存のソースがありません",
       "searchPlaceholder": "検索...",
       "addNew": "新規追加",
+      "addSource": "ソースを追加",
+      "addChunk": "チャンクを追加",
       "preLoaded": "プリロード済み",
       "private": "プライベート",
       "sync": "同期",
@@ -74,12 +76,32 @@
       "actions": "アクション",
       "view": "表示",
       "deleteWarning": "\"{{name}}\"を削除してもよろしいですか？",
-      "backToAll": "すべてのドキュメントに戻る",
+      "confirmDelete": "このファイルを削除してもよろしいですか？この操作は元に戻せません。",
+      "backToAll": "すべてのソースに戻る",
       "chunks": "チャンク",
       "noChunks": "チャンクが見つかりません",
       "noChunksAlt": "チャンクが見つかりません",
-      "goToDocuments": "ドキュメントへ移動",
-      "uploadNew": "新規アップロード"
+      "goToSources": "ソースへ移動",
+      "uploadNew": "新規アップロード",
+      "searchFiles": "ファイルを検索...",
+      "noResults": "結果が見つかりません",
+      "fileName": "名前",
+      "tokens": "トークン",
+      "size": "サイズ",
+      "fileAlt": "ファイル",
+      "folderAlt": "フォルダ",
+      "parentFolderAlt": "親フォルダ",
+      "menuAlt": "メニュー",
+      "tokensUnit": "トークン",
+      "editAlt": "編集",
+      "uploading": "アップロード中…",
+      "deleting": "削除中…",
+      "queued": "キュー: {{count}}",
+      "addFile": "ファイルを追加",
+      "uploadingFilesTitle": "ファイルをアップロード中...",
+      "deletingTitle": "削除中...",
+      "deleteDirectoryWarning": "ディレクトリ \"{{name}}\" とその内容をすべて削除してもよろしいですか？この操作は元に戻せません。",
+      "searchAlt": "検索"
     },
     "apiKeys": {
       "label": "チャットボット",
@@ -250,13 +272,14 @@
     },
     "chunk": {
       "add": "チャンクを追加",
-      "edit": "チャンクを編集",
+      "edit": "編集",
       "title": "タイトル",
       "enterTitle": "タイトルを入力",
       "bodyText": "本文",
       "promptText": "プロンプトテキスト",
-      "update": "更新",
+      "save": "保存",
       "close": "閉じる",
+      "cancel": "キャンセル",
       "delete": "削除",
       "deleteConfirmation": "このチャンクを削除してもよろしいですか？"
     }

frontend/src/locale/ru.json

@@ -52,16 +52,18 @@
       "default": "По умолчанию",
       "add": "Добавить"
     },
-    "documents": {
-      "title": "Эта таблица содержит все документы, которые доступны вам и те, которые вы загрузили",
-      "label": "Документы",
-      "name": "Название документа",
+    "sources": {
+      "title": "Здесь вы можете управлять всеми исходными файлами, которые доступны вам и которые вы загрузили.",
+      "label": "Источники",
+      "name": "Название источника",
       "date": "Дата вектора",
       "type": "Тип",
       "tokenUsage": "Использование токена",
-      "noData": "Нет существующих документов",
+      "noData": "Нет существующих источников",
       "searchPlaceholder": "Поиск...",
       "addNew": "добавить новый",
+      "addSource": "Добавить источник",
+      "addChunk": "Добавить фрагмент",
       "preLoaded": "Предзагруженный",
       "private": "Частный",
       "sync": "Синхронизация",
@@ -74,12 +76,32 @@
       "actions": "Действия",
       "view": "Просмотр",
       "deleteWarning": "Вы уверены, что хотите удалить \"{{name}}\"?",
-      "backToAll": "Вернуться ко всем документам",
+      "confirmDelete": "Вы уверены, что хотите удалить этот файл? Это действие нельзя отменить.",
+      "backToAll": "Вернуться ко всем источникам",
       "chunks": "Фрагменты",
       "noChunks": "Фрагменты не найдены",
       "noChunksAlt": "Фрагменты не найдены",
-      "goToDocuments": "Перейти к документам",
-      "uploadNew": "Загрузить новый"
+      "goToSources": "Перейти к источникам",
+      "uploadNew": "Загрузить новый",
+      "searchFiles": "Поиск файлов...",
+      "noResults": "Результаты не найдены",
+      "fileName": "Имя",
+      "tokens": "Токены",
+      "size": "Размер",
+      "fileAlt": "Файл",
+      "folderAlt": "Папка",
+      "parentFolderAlt": "Родительская папка",
+      "menuAlt": "Меню",
+      "tokensUnit": "токенов",
+      "editAlt": "Редактировать",
+      "uploading": "Загрузка…",
+      "deleting": "Удаление…",
+      "queued": "В очереди: {{count}}",
+      "addFile": "Добавить файл",
+      "uploadingFilesTitle": "Загрузка файлов...",
+      "deletingTitle": "Удаление...",
+      "deleteDirectoryWarning": "Вы уверены, что хотите удалить каталог \"{{name}}\" и все его содержимое? Это действие нельзя отменить.",
+      "searchAlt": "Поиск"
     },
     "apiKeys": {
       "label": "API ключи",
@@ -250,13 +272,14 @@
     },
     "chunk": {
       "add": "Добавить фрагмент",
-      "edit": "Редактировать фрагмент",
+      "edit": "Редактировать",
       "title": "Заголовок",
       "enterTitle": "Введите заголовок",
       "bodyText": "Текст",
       "promptText": "Текст подсказки",
-      "update": "Обновить",
+      "save": "Сохранить",
       "close": "Закрыть",
+      "cancel": "Отмена",
       "delete": "Удалить",
       "deleteConfirmation": "Вы уверены, что хотите удалить этот фрагмент?"
     }

frontend/src/locale/zh-TW.json

@@ -52,16 +52,18 @@
       "default": "預設",
       "add": "添加"
     },
-    "documents": {
-      "title": "此表格包含所有可供您使用的文件以及您上傳的文件",
-      "label": "文件",
-      "name": "文件名稱",
+    "sources": {
+      "title": "在這裡您可以管理所有可用的來源檔案以及您上傳的檔案。",
+      "label": "來源",
+      "name": "來源名稱",
       "date": "向量日期",
       "type": "類型",
       "tokenUsage": "Token 使用量",
-      "noData": "沒有現有的文件",
+      "noData": "沒有現有的來源",
       "searchPlaceholder": "搜尋...",
       "addNew": "新增文件",
+      "addSource": "新增來源",
+      "addChunk": "新增區塊",
       "preLoaded": "預載入",
       "private": "私人",
       "sync": "同步",
@@ -74,12 +76,32 @@
       "actions": "操作",
       "view": "查看",
       "deleteWarning": "您確定要刪除 \"{{name}}\" 嗎？",
-      "backToAll": "返回所有文件",
+      "confirmDelete": "您確定要刪除此檔案嗎？此操作無法復原。",
+      "backToAll": "返回所有來源",
       "chunks": "文本塊",
       "noChunks": "未找到文本塊",
       "noChunksAlt": "未找到文本塊",
-      "goToDocuments": "前往文件",
-      "uploadNew": "上傳新文件"
+      "goToSources": "前往來源",
+      "uploadNew": "上傳新文件",
+      "searchFiles": "搜尋檔案...",
+      "noResults": "未找到結果",
+      "fileName": "名稱",
+      "tokens": "Token",
+      "size": "大小",
+      "fileAlt": "檔案",
+      "folderAlt": "資料夾",
+      "parentFolderAlt": "上層資料夾",
+      "menuAlt": "選單",
+      "tokensUnit": "Token",
+      "editAlt": "編輯",
+      "uploading": "正在上傳…",
+      "deleting": "正在刪除…",
+      "queued": "已排隊：{{count}}",
+      "addFile": "新增檔案",
+      "uploadingFilesTitle": "正在上傳檔案...",
+      "deletingTitle": "正在刪除...",
+      "deleteDirectoryWarning": "您確定要刪除目錄 \"{{name}}\" 及其所有內容嗎？此操作無法復原。",
+      "searchAlt": "搜尋"
     },
     "apiKeys": {
       "label": "聊天機器人",
@@ -250,13 +272,14 @@
     },
     "chunk": {
       "add": "新增區塊",
-      "edit": "編輯區塊",
+      "edit": "編輯",
       "title": "標題",
       "enterTitle": "輸入標題",
       "bodyText": "內文",
       "promptText": "提示文字",
-      "update": "更新",
+      "save": "儲存",
       "close": "關閉",
+      "cancel": "取消",
       "delete": "刪除",
       "deleteConfirmation": "您確定要刪除此區塊嗎？"
     }

frontend/src/locale/zh.json

@@ -52,16 +52,18 @@
       "default": "默认",
       "add": "添加"
     },
-    "documents": {
-      "title": "此表格包含所有可供您使用的文档以及您上传的文档",
-      "label": "文档",
-      "name": "文件名称",
+    "sources": {
+      "title": "在这里您可以管理所有可用的源文件以及您上传的文件。",
+      "label": "来源",
+      "name": "来源名称",
       "date": "向量日期",
       "type": "类型",
       "tokenUsage": "令牌使用",
-      "noData": "没有现有的文档",
+      "noData": "没有现有的来源",
       "searchPlaceholder": "搜索...",
       "addNew": "添加新文档",
+      "addSource": "添加来源",
+      "addChunk": "添加块",
       "preLoaded": "预加载",
       "private": "私有",
       "sync": "同步",
@@ -74,12 +76,32 @@
       "actions": "操作",
       "view": "查看",
       "deleteWarning": "您确定要删除 \"{{name}}\" 吗？",
-      "backToAll": "返回所有文档",
+      "confirmDelete": "您确定要删除此文件吗？此操作无法撤销。",
+      "backToAll": "返回所有来源",
       "chunks": "文本块",
       "noChunks": "未找到文本块",
       "noChunksAlt": "未找到文本块",
-      "goToDocuments": "前往文档",
-      "uploadNew": "上传新文档"
+      "goToSources": "前往来源",
+      "uploadNew": "上传新文档",
+      "searchFiles": "搜索文件...",
+      "noResults": "未找到结果",
+      "fileName": "名称",
+      "tokens": "令牌",
+      "size": "大小",
+      "fileAlt": "文件",
+      "folderAlt": "文件夹",
+      "parentFolderAlt": "父文件夹",
+      "menuAlt": "菜单",
+      "tokensUnit": "令牌",
+      "editAlt": "编辑",
+      "uploading": "正在上传…",
+      "deleting": "正在删除…",
+      "queued": "已排队：{{count}}",
+      "addFile": "添加文件",
+      "uploadingFilesTitle": "正在上传文件...",
+      "deletingTitle": "正在删除...",
+      "deleteDirectoryWarning": "确定要删除目录 \"{{name}}\" 及其所有内容吗？此操作无法撤销。",
+      "searchAlt": "搜索"
     },
     "apiKeys": {
       "label": "聊天机器人",
@@ -250,13 +272,14 @@
     },
     "chunk": {
       "add": "添加块",
-      "edit": "编辑块",
+      "edit": "编辑",
       "title": "标题",
       "enterTitle": "输入标题",
       "bodyText": "正文",
       "promptText": "提示文本",
-      "update": "更新",
+      "save": "保存",
       "close": "关闭",
+      "cancel": "取消",
       "delete": "删除",
       "deleteConfirmation": "您确定要删除此块吗？"
     }

frontend/src/models/misc.ts

@@ -12,6 +12,7 @@ export type Doc = {
   type?: string;
   retriever?: string;
   syncFrequency?: string;
+  isNested?: boolean;
 };
 
 export type GetDocsResponse = {

frontend/src/settings/Documents.tsx

@@ -1,788 +0,0 @@
-import React, { useCallback, useEffect, useRef, useState } from 'react';
-import { useTranslation } from 'react-i18next';
-import { useDispatch, useSelector } from 'react-redux';
-
-import userService from '../api/services/userService';
-import ArrowLeft from '../assets/arrow-left.svg';
-import caretSort from '../assets/caret-sort.svg';
-import Edit from '../assets/edit.svg';
-import EyeView from '../assets/eye-view.svg';
-import NoFilesDarkIcon from '../assets/no-files-dark.svg';
-import NoFilesIcon from '../assets/no-files.svg';
-import Trash from '../assets/red-trash.svg';
-import SyncIcon from '../assets/sync.svg';
-import ThreeDots from '../assets/three-dots.svg';
-import ContextMenu, { MenuOption } from '../components/ContextMenu';
-import Pagination from '../components/DocumentPagination';
-import DropdownMenu from '../components/DropdownMenu';
-import Input from '../components/Input';
-import SkeletonLoader from '../components/SkeletonLoader';
-import Spinner from '../components/Spinner';
-import { useDarkTheme, useLoaderState } from '../hooks';
-import ChunkModal from '../modals/ChunkModal';
-import ConfirmationModal from '../modals/ConfirmationModal';
-import { ActiveState, Doc, DocumentsProps } from '../models/misc';
-import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi';
-import {
-  selectToken,
-  setPaginatedDocuments,
-  setSourceDocs,
-} from '../preferences/preferenceSlice';
-import Upload from '../upload/Upload';
-import { formatDate } from '../utils/dateTimeUtils';
-import { ChunkType } from './types';
-
-const formatTokens = (tokens: number): string => {
-  const roundToTwoDecimals = (num: number): string => {
-    return (Math.round((num + Number.EPSILON) * 100) / 100).toString();
-  };
-
-  if (tokens >= 1_000_000_000) {
-    return roundToTwoDecimals(tokens / 1_000_000_000) + 'b';
-  } else if (tokens >= 1_000_000) {
-    return roundToTwoDecimals(tokens / 1_000_000) + 'm';
-  } else if (tokens >= 1_000) {
-    return roundToTwoDecimals(tokens / 1_000) + 'k';
-  } else {
-    return tokens.toString();
-  }
-};
-
-export default function Documents({
-  paginatedDocuments,
-  handleDeleteDocument,
-}: DocumentsProps) {
-  const { t } = useTranslation();
-  const dispatch = useDispatch();
-  const token = useSelector(selectToken);
-
-  const [searchTerm, setSearchTerm] = useState<string>('');
-  const [modalState, setModalState] = useState<ActiveState>('INACTIVE');
-  const [isOnboarding, setIsOnboarding] = useState<boolean>(false);
-  const [loading, setLoading] = useLoaderState(false);
-  const [sortField, setSortField] = useState<'date' | 'tokens'>('date');
-  const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('desc');
-  // Pagination
-  const [currentPage, setCurrentPage] = useState<number>(1);
-  const [rowsPerPage, setRowsPerPage] = useState<number>(10);
-  const [totalPages, setTotalPages] = useState<number>(1);
-
-  const [activeMenuId, setActiveMenuId] = useState<string | null>(null);
-  const menuRefs = useRef<{
-    [key: string]: React.RefObject<HTMLDivElement | null>;
-  }>({});
-
-  // Create or get a ref for each document wrapper div (not the td)
-  const getMenuRef = (docId: string) => {
-    if (!menuRefs.current[docId]) {
-      menuRefs.current[docId] = React.createRef<HTMLDivElement>();
-    }
-    return menuRefs.current[docId];
-  };
-
-  const handleMenuClick = (e: React.MouseEvent, docId: string) => {
-    e.preventDefault();
-    e.stopPropagation();
-
-    const isAnyMenuOpen =
-      (syncMenuState.isOpen && syncMenuState.docId === docId) ||
-      activeMenuId === docId;
-
-    if (isAnyMenuOpen) {
-      setSyncMenuState((prev) => ({ ...prev, isOpen: false, docId: null }));
-      setActiveMenuId(null);
-      return;
-    }
-    setActiveMenuId(docId);
-  };
-
-  useEffect(() => {
-    const handleClickOutside = (event: MouseEvent) => {
-      if (activeMenuId) {
-        const activeRef = menuRefs.current[activeMenuId];
-        if (
-          activeRef?.current &&
-          !activeRef.current.contains(event.target as Node)
-        ) {
-          setActiveMenuId(null);
-        }
-      }
-    };
-
-    document.addEventListener('mousedown', handleClickOutside);
-    return () => document.removeEventListener('mousedown', handleClickOutside);
-  }, [activeMenuId]);
-
-  const currentDocuments = paginatedDocuments ?? [];
-  const syncOptions = [
-    { label: t('settings.documents.syncFrequency.never'), value: 'never' },
-    { label: t('settings.documents.syncFrequency.daily'), value: 'daily' },
-    { label: t('settings.documents.syncFrequency.weekly'), value: 'weekly' },
-    { label: t('settings.documents.syncFrequency.monthly'), value: 'monthly' },
-  ];
-  const [showDocumentChunks, setShowDocumentChunks] = useState<Doc>();
-  const [isMenuOpen, setIsMenuOpen] = useState(false);
-  const [syncMenuState, setSyncMenuState] = useState<{
-    isOpen: boolean;
-    docId: string | null;
-    document: Doc | null;
-  }>({
-    isOpen: false,
-    docId: null,
-    document: null,
-  });
-
-  const refreshDocs = useCallback(
-    (
-      field: 'date' | 'tokens' | undefined,
-      pageNumber?: number,
-      rows?: number,
-    ) => {
-      const page = pageNumber ?? currentPage;
-      const rowsPerPg = rows ?? rowsPerPage;
-
-      // If field is undefined, (Pagination or Search) use the current sortField
-      const newSortField = field ?? sortField;
-
-      // If field is undefined, (Pagination or Search) use the current sortOrder
-      const newSortOrder =
-        field === sortField
-          ? sortOrder === 'asc'
-            ? 'desc'
-            : 'asc'
-          : sortOrder;
-
-      // If field is defined, update the sortField and sortOrder
-      if (field) {
-        setSortField(newSortField);
-        setSortOrder(newSortOrder);
-      }
-
-      setLoading(true);
-      getDocsWithPagination(
-        newSortField,
-        newSortOrder,
-        page,
-        rowsPerPg,
-        searchTerm,
-        token,
-      )
-        .then((data) => {
-          dispatch(setPaginatedDocuments(data ? data.docs : []));
-          setTotalPages(data ? data.totalPages : 0);
-        })
-        .catch((error) => console.error(error))
-        .finally(() => {
-          setLoading(false);
-        });
-    },
-    [currentPage, rowsPerPage, sortField, sortOrder, searchTerm],
-  );
-
-  const handleManageSync = (doc: Doc, sync_frequency: string) => {
-    setLoading(true);
-    userService
-      .manageSync({ source_id: doc.id, sync_frequency }, token)
-      .then(() => {
-        return getDocs(token);
-      })
-      .then((data) => {
-        dispatch(setSourceDocs(data));
-        return getDocsWithPagination(
-          sortField,
-          sortOrder,
-          currentPage,
-          rowsPerPage,
-          searchTerm,
-          token,
-        );
-      })
-      .then((paginatedData) => {
-        dispatch(
-          setPaginatedDocuments(paginatedData ? paginatedData.docs : []),
-        );
-        setTotalPages(paginatedData ? paginatedData.totalPages : 0);
-      })
-      .catch((error) => console.error('Error in handleManageSync:', error))
-      .finally(() => {
-        setLoading(false);
-      });
-  };
-
-  const [documentToDelete, setDocumentToDelete] = useState<{
-    index: number;
-    document: Doc;
-  } | null>(null);
-  const [deleteModalState, setDeleteModalState] =
-    useState<ActiveState>('INACTIVE');
-
-  const handleDeleteConfirmation = (index: number, document: Doc) => {
-    setDocumentToDelete({ index, document });
-    setDeleteModalState('ACTIVE');
-  };
-
-  const handleConfirmedDelete = () => {
-    if (documentToDelete) {
-      handleDeleteDocument(documentToDelete.index, documentToDelete.document);
-      setDeleteModalState('INACTIVE');
-      setDocumentToDelete(null);
-    }
-  };
-
-  const getActionOptions = (index: number, document: Doc): MenuOption[] => {
-    const actions: MenuOption[] = [
-      {
-        icon: EyeView,
-        label: t('settings.documents.view'),
-        onClick: () => {
-          setShowDocumentChunks(document);
-        },
-        iconWidth: 18,
-        iconHeight: 18,
-        variant: 'primary',
-      },
-    ];
-
-    if (document.syncFrequency) {
-      actions.push({
-        icon: SyncIcon,
-        label: t('settings.documents.sync'),
-        onClick: () => {
-          setSyncMenuState({
-            isOpen: true,
-            docId: document.id ?? null,
-            document: document,
-          });
-        },
-        iconWidth: 14,
-        iconHeight: 14,
-        variant: 'primary',
-      });
-    }
-
-    actions.push({
-      icon: Trash,
-      label: t('convTile.delete'),
-      onClick: () => {
-        handleDeleteConfirmation(index, document);
-      },
-      iconWidth: 18,
-      iconHeight: 18,
-      variant: 'danger',
-    });
-
-    return actions;
-  };
-  useEffect(() => {
-    refreshDocs(undefined, 1, rowsPerPage);
-  }, [searchTerm]);
-
-  return showDocumentChunks ? (
-    <DocumentChunks
-      document={showDocumentChunks}
-      handleGoBack={() => {
-        setShowDocumentChunks(undefined);
-      }}
-    />
-  ) : (
-    <div className="mt-8 flex w-full max-w-full flex-col overflow-hidden">
-      <div className="relative flex grow flex-col">
-        <div className="mb-6">
-          <h2 className="text-sonic-silver text-base font-medium">
-            {t('settings.documents.title')}
-          </h2>
-        </div>
-        <div className="mb-6 flex flex-col items-start justify-between gap-3 sm:flex-row sm:items-center">
-          <div className="w-full sm:w-auto">
-            <label htmlFor="document-search-input" className="sr-only">
-              {t('settings.documents.searchPlaceholder')}
-            </label>
-            <Input
-              maxLength={256}
-              placeholder={t('settings.documents.searchPlaceholder')}
-              name="Document-search-input"
-              type="text"
-              id="document-search-input"
-              value={searchTerm}
-              onChange={(e) => {
-                setSearchTerm(e.target.value);
-                setCurrentPage(1);
-              }}
-              borderVariant="thin"
-            />
-          </div>
-          <button
-            className="bg-purple-30 hover:bg-violets-are-blue flex h-[32px] min-w-[108px] items-center justify-center rounded-full px-4 text-sm whitespace-normal text-white"
-            title={t('settings.documents.addNew')}
-            onClick={() => {
-              setIsOnboarding(false);
-              setModalState('ACTIVE');
-            }}
-          >
-            {t('settings.documents.addNew')}
-          </button>
-        </div>
-        <div className="relative w-full">
-          <div className="dark:border-silver/40 overflow-hidden rounded-md border border-gray-300">
-            <div className="table-scroll overflow-x-auto">
-              <table className="w-full table-auto">
-                <thead>
-                  <tr className="dark:border-silver/40 border-b border-gray-300">
-                    <th className="text-sonic-silver w-[45%] px-4 py-3 text-left text-xs font-medium">
-                      {t('settings.documents.name')}
-                    </th>
-                    <th className="text-sonic-silver w-[30%] px-4 py-3 text-left text-xs font-medium">
-                      <div className="flex items-center justify-start">
-                        {t('settings.documents.date')}
-                        <img
-                          className="ml-2 cursor-pointer"
-                          onClick={() => refreshDocs('date')}
-                          src={caretSort}
-                          alt="sort"
-                        />
-                      </div>
-                    </th>
-                    <th className="text-sonic-silver w-[15%] px-4 py-3 text-left text-xs font-medium">
-                      <div className="flex items-center justify-start">
-                        <span className="hidden sm:inline">
-                          {t('settings.documents.tokenUsage')}
-                        </span>
-                        <span className="sm:hidden">
-                          {t('settings.documents.tokenUsage')}
-                        </span>
-                        <img
-                          className="ml-2 cursor-pointer"
-                          onClick={() => refreshDocs('tokens')}
-                          src={caretSort}
-                          alt="sort"
-                        />
-                      </div>
-                    </th>
-                    <th className="sr-only w-[10%] px-4 py-3">
-                      {t('settings.documents.actions')}
-                    </th>
-                  </tr>
-                </thead>
-                <tbody className="dark:divide-silver/40 divide-y divide-gray-300">
-                  {loading ? (
-                    <SkeletonLoader component="table" />
-                  ) : !currentDocuments?.length ? (
-                    <tr>
-                      <td
-                        colSpan={4}
-                        className="bg-transparent py-4 text-center text-gray-700 dark:text-neutral-200"
-                      >
-                        {t('settings.documents.noData')}
-                      </td>
-                    </tr>
-                  ) : (
-                    currentDocuments.map((document, index) => {
-                      const docId = document.id ? document.id.toString() : '';
-
-                      return (
-                        <tr key={docId} className="group transition-colors">
-                          <td
-                            className="max-w-0 min-w-48 truncate px-4 py-4 text-sm font-semibold text-gray-700 group-hover:bg-gray-50 dark:text-[#E0E0E0] dark:group-hover:bg-gray-800/50"
-                            title={document.name}
-                          >
-                            {document.name}
-                          </td>
-                          <td className="px-4 py-4 text-sm whitespace-nowrap text-gray-700 group-hover:bg-gray-50 dark:text-[#E0E0E0] dark:group-hover:bg-gray-800/50">
-                            {document.date ? formatDate(document.date) : ''}
-                          </td>
-                          <td className="px-4 py-4 text-sm whitespace-nowrap text-gray-700 group-hover:bg-gray-50 dark:text-[#E0E0E0] dark:group-hover:bg-gray-800/50">
-                            {document.tokens
-                              ? formatTokens(+document.tokens)
-                              : ''}
-                          </td>
-                          <td
-                            className="px-4 py-4 text-right group-hover:bg-gray-50 dark:group-hover:bg-gray-800/50"
-                            onClick={(e) => e.stopPropagation()}
-                          >
-                            <div
-                              ref={getMenuRef(docId)}
-                              className="relative flex items-center justify-end gap-3"
-                            >
-                              {document.syncFrequency && (
-                                <DropdownMenu
-                                  name={t('settings.documents.sync')}
-                                  options={syncOptions}
-                                  onSelect={(value: string) => {
-                                    handleManageSync(document, value);
-                                  }}
-                                  defaultValue={document.syncFrequency}
-                                  icon={SyncIcon}
-                                  isOpen={
-                                    syncMenuState.docId === docId &&
-                                    syncMenuState.isOpen
-                                  }
-                                  onOpenChange={(isOpen) => {
-                                    setSyncMenuState((prev) => ({
-                                      ...prev,
-                                      isOpen,
-                                      docId: isOpen ? docId : null,
-                                      document: isOpen ? document : null,
-                                    }));
-                                  }}
-                                  anchorRef={getMenuRef(docId)}
-                                  position="bottom-left"
-                                  offset={{ x: 24, y: -24 }}
-                                  className="min-w-[120px]"
-                                />
-                              )}
-                              <button
-                                onClick={(e) => handleMenuClick(e, docId)}
-                                className="inline-flex h-8 w-8 shrink-0 items-center justify-center rounded-full transition-colors hover:bg-gray-100 dark:hover:bg-gray-700"
-                                aria-label="Open menu"
-                                data-testid={`menu-button-${docId}`}
-                              >
-                                <img
-                                  src={ThreeDots}
-                                  alt={t('convTile.menu')}
-                                  className="h-4 w-4 opacity-60 hover:opacity-100"
-                                />
-                              </button>
-                              <ContextMenu
-                                isOpen={activeMenuId === docId}
-                                setIsOpen={(isOpen) => {
-                                  setActiveMenuId(isOpen ? docId : null);
-                                }}
-                                options={getActionOptions(index, document)}
-                                anchorRef={getMenuRef(docId)}
-                                position="bottom-left"
-                                offset={{ x: 48, y: 0 }}
-                                className="z-50"
-                              />
-                            </div>
-                          </td>
-                        </tr>
-                      );
-                    })
-                  )}
-                </tbody>
-              </table>
-            </div>
-          </div>
-        </div>
-      </div>
-
-      <div className="mt-auto pt-4">
-        <Pagination
-          currentPage={currentPage}
-          totalPages={totalPages}
-          rowsPerPage={rowsPerPage}
-          onPageChange={(page) => {
-            setCurrentPage(page);
-            refreshDocs(undefined, page, rowsPerPage);
-          }}
-          onRowsPerPageChange={(rows) => {
-            setRowsPerPage(rows);
-            setCurrentPage(1);
-            refreshDocs(undefined, 1, rows);
-          }}
-        />
-      </div>
-
-      {modalState === 'ACTIVE' && (
-        <Upload
-          receivedFile={[]}
-          setModalState={setModalState}
-          isOnboarding={isOnboarding}
-          renderTab={null}
-          close={() => setModalState('INACTIVE')}
-          onSuccessfulUpload={() =>
-            refreshDocs(undefined, currentPage, rowsPerPage)
-          }
-        />
-      )}
-
-      {deleteModalState === 'ACTIVE' && documentToDelete && (
-        <ConfirmationModal
-          message={t('settings.documents.deleteWarning', {
-            name: documentToDelete.document.name,
-          })}
-          modalState={deleteModalState}
-          setModalState={setDeleteModalState}
-          handleSubmit={handleConfirmedDelete}
-          handleCancel={() => {
-            setDeleteModalState('INACTIVE');
-            setDocumentToDelete(null);
-          }}
-          submitLabel={t('convTile.delete')}
-          variant="danger"
-        />
-      )}
-    </div>
-  );
-}
-
-function DocumentChunks({
-  document,
-  handleGoBack,
-}: {
-  document: Doc;
-  handleGoBack: () => void;
-}) {
-  const { t } = useTranslation();
-  const token = useSelector(selectToken);
-  const [isDarkTheme] = useDarkTheme();
-  const [paginatedChunks, setPaginatedChunks] = useState<ChunkType[]>([]);
-  const [page, setPage] = useState(1);
-  const [perPage, setPerPage] = useState(5);
-  const [totalChunks, setTotalChunks] = useState(0);
-  const [loading, setLoading] = useLoaderState(true);
-  const [searchTerm, setSearchTerm] = useState<string>('');
-  const [addModal, setAddModal] = useState<ActiveState>('INACTIVE');
-  const [editModal, setEditModal] = useState<{
-    state: ActiveState;
-    chunk: ChunkType | null;
-  }>({ state: 'INACTIVE', chunk: null });
-
-  const fetchChunks = () => {
-    setLoading(true);
-    try {
-      userService
-        .getDocumentChunks(document.id ?? '', page, perPage, token)
-        .then((response) => {
-          if (!response.ok) {
-            setLoading(false);
-            setPaginatedChunks([]);
-            throw new Error('Failed to fetch chunks data');
-          }
-          return response.json();
-        })
-        .then((data) => {
-          setPage(data.page);
-          setPerPage(data.per_page);
-          setTotalChunks(data.total);
-          setPaginatedChunks(data.chunks);
-          setLoading(false);
-        });
-    } catch (e) {
-      console.log(e);
-      setLoading(false);
-    }
-  };
-
-  const handleAddChunk = (title: string, text: string) => {
-    try {
-      userService
-        .addChunk(
-          {
-            id: document.id ?? '',
-            text: text,
-            metadata: {
-              title: title,
-            },
-          },
-          token,
-        )
-        .then((response) => {
-          if (!response.ok) {
-            throw new Error('Failed to add chunk');
-          }
-          fetchChunks();
-        });
-    } catch (e) {
-      console.log(e);
-    }
-  };
-
-  const handleUpdateChunk = (title: string, text: string, chunk: ChunkType) => {
-    try {
-      userService
-        .updateChunk(
-          {
-            id: document.id ?? '',
-            chunk_id: chunk.doc_id,
-            text: text,
-            metadata: {
-              title: title,
-            },
-          },
-          token,
-        )
-        .then((response) => {
-          if (!response.ok) {
-            throw new Error('Failed to update chunk');
-          }
-          fetchChunks();
-        });
-    } catch (e) {
-      console.log(e);
-    }
-  };
-
-  const handleDeleteChunk = (chunk: ChunkType) => {
-    try {
-      userService
-        .deleteChunk(document.id ?? '', chunk.doc_id, token)
-        .then((response) => {
-          if (!response.ok) {
-            throw new Error('Failed to delete chunk');
-          }
-          setEditModal({ state: 'INACTIVE', chunk: null });
-          fetchChunks();
-        });
-    } catch (e) {
-      console.log(e);
-    }
-  };
-
-  React.useEffect(() => {
-    fetchChunks();
-  }, [page, perPage]);
-  return (
-    <div className="mt-8 flex flex-col">
-      <div className="text-eerie-black dark:text-bright-gray mb-3 flex items-center gap-3 text-sm">
-        <button
-          className="rounded-full border p-3 text-sm text-gray-400 dark:border-0 dark:bg-[#28292D] dark:text-gray-500 dark:hover:bg-[#2E2F34]"
-          onClick={handleGoBack}
-        >
-          <img src={ArrowLeft} alt="left-arrow" className="h-3 w-3" />
-        </button>
-        <p className="mt-px">{t('settings.documents.backToAll')}</p>
-      </div>
-      <div className="my-3 flex items-center justify-between gap-1">
-        <div className="text-eerie-black dark:text-bright-gray flex w-full items-center gap-2 sm:w-auto">
-          <p className="hidden text-2xl font-semibold sm:flex">{`${totalChunks} ${t('settings.documents.chunks')}`}</p>
-          <label htmlFor="chunk-search-input" className="sr-only">
-            {t('settings.documents.searchPlaceholder')}
-          </label>
-          <Input
-            maxLength={256}
-            placeholder={t('settings.documents.searchPlaceholder')}
-            name="chunk-search-input"
-            type="text"
-            id="chunk-search-input"
-            value={searchTerm}
-            onChange={(e) => {
-              setSearchTerm(e.target.value);
-            }}
-            borderVariant="thin"
-          />
-        </div>
-        <button
-          className="bg-purple-30 hover:bg-violets-are-blue flex h-[32px] min-w-[108px] items-center justify-center rounded-full px-4 text-sm whitespace-normal text-white"
-          title={t('settings.documents.addNew')}
-          onClick={() => setAddModal('ACTIVE')}
-        >
-          {t('settings.documents.addNew')}
-        </button>
-      </div>
-      {loading ? (
-        <div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
-          <div className="col-span-2 mt-24 flex h-32 items-center justify-center lg:col-span-3">
-            <Spinner />
-          </div>
-        </div>
-      ) : (
-        <div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
-          {paginatedChunks.filter((chunk) => {
-            if (!chunk.metadata?.title) return true;
-            return chunk.metadata.title
-              .toLowerCase()
-              .includes(searchTerm.toLowerCase());
-          }).length === 0 ? (
-            <div className="col-span-2 mt-24 text-center text-gray-500 lg:col-span-3 dark:text-gray-400">
-              <img
-                src={isDarkTheme ? NoFilesDarkIcon : NoFilesIcon}
-                alt={t('settings.documents.noChunksAlt')}
-                className="mx-auto mb-2 h-24 w-24"
-              />
-              {t('settings.documents.noChunks')}
-            </div>
-          ) : (
-            paginatedChunks
-              .filter((chunk) => {
-                if (!chunk.metadata?.title) return true;
-                return chunk.metadata.title
-                  .toLowerCase()
-                  .includes(searchTerm.toLowerCase());
-              })
-              .map((chunk, index) => (
-                <div
-                  key={index}
-                  className="border-silver dark:border-silver/40 relative flex h-56 w-full flex-col justify-between rounded-2xl border p-6"
-                >
-                  <div className="w-full">
-                    <div className="flex w-full items-center justify-between">
-                      <button
-                        aria-label={'edit'}
-                        onClick={() => {
-                          setEditModal({
-                            state: 'ACTIVE',
-                            chunk: chunk,
-                          });
-                        }}
-                        className="absolute top-3 right-3 h-4 w-4 cursor-pointer"
-                      >
-                        <img
-                          alt={'edit'}
-                          src={Edit}
-                          className="opacity-60 hover:opacity-100"
-                        />
-                      </button>
-                    </div>
-                    <div className="mt-[9px]">
-                      <p className="ellipsis-text text-eerie-black h-12 text-sm leading-relaxed font-semibold break-words dark:text-[#EEEEEE]">
-                        {chunk.metadata?.title ?? 'Untitled'}
-                      </p>
-                      <p className="mt-1 h-[110px] overflow-y-auto pr-1 text-[13px] leading-relaxed break-words text-gray-600 dark:text-gray-400">
-                        {chunk.text}
-                      </p>
-                    </div>
-                  </div>
-                </div>
-              ))
-          )}
-        </div>
-      )}
-      {!loading &&
-        paginatedChunks.filter((chunk) => {
-          if (!chunk.metadata?.title) return true;
-          return chunk.metadata.title
-            .toLowerCase()
-            .includes(searchTerm.toLowerCase());
-        }).length !== 0 && (
-          <div className="mt-10 flex w-full items-center justify-center">
-            <Pagination
-              currentPage={page}
-              totalPages={Math.ceil(totalChunks / perPage)}
-              rowsPerPage={perPage}
-              onPageChange={(page) => {
-                setPage(page);
-              }}
-              onRowsPerPageChange={(rows) => {
-                setPerPage(rows);
-                setPage(1);
-              }}
-            />
-          </div>
-        )}
-      <ChunkModal
-        type="ADD"
-        modalState={addModal}
-        setModalState={setAddModal}
-        handleSubmit={handleAddChunk}
-      />
-      {editModal.chunk && (
-        <ChunkModal
-          type="EDIT"
-          modalState={editModal.state}
-          setModalState={(state) =>
-            setEditModal((prev) => ({ ...prev, state }))
-          }
-          handleSubmit={(title, text) => {
-            handleUpdateChunk(title, text, editModal.chunk as ChunkType);
-          }}
-          originalText={editModal.chunk?.text ?? ''}
-          originalTitle={editModal.chunk?.metadata?.title ?? ''}
-          handleDelete={() => {
-            handleDeleteChunk(editModal.chunk as ChunkType);
-          }}
-        />
-      )}
-    </div>
-  );
-}

frontend/src/settings/Sources.tsx

@@ -0,0 +1,510 @@
+
+import React, { useCallback, useEffect, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { useDispatch, useSelector } from 'react-redux';
+
+import userService from '../api/services/userService';
+
+import EyeView from '../assets/eye-view.svg';
+import NoFilesIcon from '../assets/no-files.svg';
+import NoFilesDarkIcon from '../assets/no-files-dark.svg';
+import Trash from '../assets/red-trash.svg';
+import SyncIcon from '../assets/sync.svg';
+import ThreeDots from '../assets/three-dots.svg';
+import CalendarIcon from '../assets/calendar.svg';
+import DiscIcon from '../assets/disc.svg';
+import ContextMenu, { MenuOption } from '../components/ContextMenu';
+import Pagination from '../components/DocumentPagination';
+import DropdownMenu from '../components/DropdownMenu';
+import SkeletonLoader from '../components/SkeletonLoader';
+import { useDarkTheme, useLoaderState } from '../hooks';
+import ConfirmationModal from '../modals/ConfirmationModal';
+import { ActiveState, Doc, DocumentsProps } from '../models/misc';
+import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi';
+import {
+  selectToken,
+  setPaginatedDocuments,
+  setSourceDocs,
+} from '../preferences/preferenceSlice';
+import Upload from '../upload/Upload';
+import { formatDate } from '../utils/dateTimeUtils';
+import FileTreeComponent from '../components/FileTreeComponent';
+import Chunks from '../components/Chunks';
+
+const formatTokens = (tokens: number): string => {
+  const roundToTwoDecimals = (num: number): string => {
+    return (Math.round((num + Number.EPSILON) * 100) / 100).toString();
+  };
+
+  if (tokens >= 1_000_000_000) {
+    return roundToTwoDecimals(tokens / 1_000_000_000) + 'b';
+  } else if (tokens >= 1_000_000) {
+    return roundToTwoDecimals(tokens / 1_000_000) + 'm';
+  } else if (tokens >= 1_000) {
+    return roundToTwoDecimals(tokens / 1_000) + 'k';
+  } else {
+    return tokens.toString();
+  }
+};
+
+export default function Sources({
+  paginatedDocuments,
+  handleDeleteDocument,
+}: DocumentsProps) {
+  const { t } = useTranslation();
+  const [isDarkTheme] = useDarkTheme();
+  const dispatch = useDispatch();
+  const token = useSelector(selectToken);
+
+  const [searchTerm, setSearchTerm] = useState<string>('');
+  const [debouncedSearchTerm, setDebouncedSearchTerm] = useState<string>('');
+  const [modalState, setModalState] = useState<ActiveState>('INACTIVE');
+  const [isOnboarding, setIsOnboarding] = useState<boolean>(false);
+  const [loading, setLoading] = useLoaderState(false);
+  const [sortField, setSortField] = useState<'date' | 'tokens'>('date');
+  const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('desc');
+  // Pagination
+  const [currentPage, setCurrentPage] = useState<number>(1);
+  const [rowsPerPage, setRowsPerPage] = useState<number>(10);
+  const [totalPages, setTotalPages] = useState<number>(1);
+
+  const [activeMenuId, setActiveMenuId] = useState<string | null>(null);
+  const menuRefs = useRef<{
+    [key: string]: React.RefObject<HTMLDivElement | null>;
+  }>({});
+
+  // Create or get a ref for each document wrapper div (not the td)
+  const getMenuRef = (docId: string) => {
+    if (!menuRefs.current[docId]) {
+      menuRefs.current[docId] = React.createRef<HTMLDivElement>();
+    }
+    return menuRefs.current[docId];
+  };
+
+  const handleMenuClick = (e: React.MouseEvent, docId: string) => {
+    e.preventDefault();
+    e.stopPropagation();
+
+    const isAnyMenuOpen =
+      (syncMenuState.isOpen && syncMenuState.docId === docId) ||
+      activeMenuId === docId;
+
+    if (isAnyMenuOpen) {
+      setSyncMenuState((prev) => ({ ...prev, isOpen: false, docId: null }));
+      setActiveMenuId(null);
+      return;
+    }
+    setActiveMenuId(docId);
+  };
+
+  const currentDocuments = paginatedDocuments ?? [];
+  const syncOptions = [
+    { label: t('settings.sources.syncFrequency.never'), value: 'never' },
+    { label: t('settings.sources.syncFrequency.daily'), value: 'daily' },
+    { label: t('settings.sources.syncFrequency.weekly'), value: 'weekly' },
+    { label: t('settings.sources.syncFrequency.monthly'), value: 'monthly' },
+  ];
+  const [documentToView, setDocumentToView] = useState<Doc>();
+  const [isMenuOpen, setIsMenuOpen] = useState(false);
+  const [syncMenuState, setSyncMenuState] = useState<{
+    isOpen: boolean;
+    docId: string | null;
+    document: Doc | null;
+  }>({
+    isOpen: false,
+    docId: null,
+    document: null,
+  });
+
+  useEffect(() => {
+    const timer = setTimeout(() => {
+      setDebouncedSearchTerm(searchTerm);
+    }, 500);
+
+    return () => clearTimeout(timer);
+  }, [searchTerm]);
+
+  const refreshDocs = useCallback(
+    (
+      field: 'date' | 'tokens' | undefined,
+      pageNumber?: number,
+      rows?: number,
+    ) => {
+      const page = pageNumber ?? currentPage;
+      const rowsPerPg = rows ?? rowsPerPage;
+
+      // If field is undefined, (Pagination or Search) use the current sortField
+      const newSortField = field ?? sortField;
+
+      // If field is undefined, (Pagination or Search) use the current sortOrder
+      const newSortOrder =
+        field === sortField
+          ? sortOrder === 'asc'
+            ? 'desc'
+            : 'asc'
+          : sortOrder;
+
+      // If field is defined, update the sortField and sortOrder
+      if (field) {
+        setSortField(newSortField);
+        setSortOrder(newSortOrder);
+      }
+
+      setLoading(true);
+      getDocsWithPagination(
+        newSortField,
+        newSortOrder,
+        page,
+        rowsPerPg,
+        debouncedSearchTerm,
+        token,
+      )
+        .then((data) => {
+          dispatch(setPaginatedDocuments(data ? data.docs : []));
+          setTotalPages(data ? data.totalPages : 0);
+        })
+        .catch((error) => console.error(error))
+        .finally(() => {
+          setLoading(false);
+        });
+    },
+    [currentPage, rowsPerPage, sortField, sortOrder, debouncedSearchTerm],
+  );
+
+  const handleManageSync = (doc: Doc, sync_frequency: string) => {
+    setLoading(true);
+    userService
+      .manageSync({ source_id: doc.id, sync_frequency }, token)
+      .then(() => {
+        return getDocs(token);
+      })
+      .then((data) => {
+        dispatch(setSourceDocs(data));
+        return getDocsWithPagination(
+          sortField,
+          sortOrder,
+          currentPage,
+          rowsPerPage,
+          searchTerm,
+          token,
+        );
+      })
+      .then((paginatedData) => {
+        dispatch(
+          setPaginatedDocuments(paginatedData ? paginatedData.docs : []),
+        );
+        setTotalPages(paginatedData ? paginatedData.totalPages : 0);
+      })
+      .catch((error) => console.error('Error in handleManageSync:', error))
+      .finally(() => {
+        setLoading(false);
+      });
+  };
+
+  const [documentToDelete, setDocumentToDelete] = useState<{
+    index: number;
+    document: Doc;
+  } | null>(null);
+  const [deleteModalState, setDeleteModalState] =
+    useState<ActiveState>('INACTIVE');
+
+  const handleDeleteConfirmation = (index: number, document: Doc) => {
+    setDocumentToDelete({ index, document });
+    setDeleteModalState('ACTIVE');
+  };
+
+  const handleConfirmedDelete = () => {
+    if (documentToDelete) {
+      handleDeleteDocument(documentToDelete.index, documentToDelete.document);
+      setDeleteModalState('INACTIVE');
+      setDocumentToDelete(null);
+    }
+  };
+
+  const getActionOptions = (index: number, document: Doc): MenuOption[] => {
+    const actions: MenuOption[] = [
+      {
+        icon: EyeView,
+        label: t('settings.sources.view'),
+        onClick: () => {
+          setDocumentToView(document);
+        },
+        iconWidth: 18,
+        iconHeight: 18,
+        variant: 'primary',
+      },
+    ];
+
+    if (document.syncFrequency) {
+      actions.push({
+        icon: SyncIcon,
+        label: t('settings.sources.sync'),
+        onClick: () => {
+          setSyncMenuState({
+            isOpen: true,
+            docId: document.id ?? null,
+            document: document,
+          });
+        },
+        iconWidth: 14,
+        iconHeight: 14,
+        variant: 'primary',
+      });
+    }
+
+    actions.push({
+      icon: Trash,
+      label: t('convTile.delete'),
+      onClick: () => {
+        handleDeleteConfirmation(index, document);
+      },
+      iconWidth: 18,
+      iconHeight: 18,
+      variant: 'danger',
+    });
+
+    return actions;
+  };
+  useEffect(() => {
+    refreshDocs(undefined, 1, rowsPerPage);
+  }, [debouncedSearchTerm]);
+
+  return documentToView ? (
+    <div className="mt-8 flex flex-col">
+      {documentToView.isNested ? (
+        <FileTreeComponent
+          docId={documentToView.id || ''}
+          sourceName={documentToView.name}
+          onBackToDocuments={() => setDocumentToView(undefined)}
+        />
+      ) : (
+        <Chunks
+          documentId={documentToView.id || ''}
+          documentName={documentToView.name}
+          handleGoBack={() => setDocumentToView(undefined)}
+        />
+      )}
+    </div>
+  ) : (
+    <div className="mt-8 flex w-full max-w-full flex-col overflow-hidden">
+      <div className="relative flex grow flex-col">
+        <div className="mb-6">
+          <h2 className="text-sonic-silver text-base font-medium">
+            {t('settings.sources.title')}
+          </h2>
+        </div>
+        <div className="mb-6 flex flex-col items-start justify-between gap-3 sm:flex-row sm:items-center">
+          <div className="w-full sm:w-auto">
+            <label htmlFor="document-search-input" className="sr-only">
+              {t('settings.sources.searchPlaceholder')}
+            </label>
+            <div className="relative w-[280px]">
+              <input
+                maxLength={256}
+                placeholder={t('settings.sources.searchPlaceholder')}
+                name="Document-search-input"
+                type="text"
+                id="document-search-input"
+                value={searchTerm}
+                onChange={(e) => {
+                  setSearchTerm(e.target.value);
+                  setCurrentPage(1);
+                }}
+                className="w-full h-[32px] rounded-full border border-silver dark:border-silver/40 bg-transparent px-3 text-sm text-jet dark:text-bright-gray placeholder:text-gray-400 dark:placeholder:text-gray-500 outline-none focus:border-silver dark:focus:border-silver/60"
+              />
+            </div>
+          </div>
+          <button
+            className="bg-purple-30 hover:bg-violets-are-blue flex h-[38px] min-w-[108px] items-center justify-center rounded-full px-4 text-[14px] whitespace-normal text-white"
+            title={t('settings.sources.addSource')}
+            onClick={() => {
+              setIsOnboarding(false);
+              setModalState('ACTIVE');
+            }}
+          >
+            {t('settings.sources.addSource')}
+          </button>
+        </div>
+        <div className="relative w-full">
+          {loading ? (
+            <div className="w-full grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-6 px-2 py-4">
+              <SkeletonLoader component="sourceCards" count={rowsPerPage} />
+            </div>
+          ) : !currentDocuments?.length ? (
+            <div className="flex flex-col items-center justify-center py-12">
+              <img
+                src={isDarkTheme ? NoFilesDarkIcon : NoFilesIcon}
+                alt={t('settings.sources.noData')}
+                className="mx-auto mb-6 h-32 w-32"
+              />
+              <p className="text-center text-lg text-gray-500 dark:text-gray-400">
+                {t('settings.sources.noData')}
+              </p>
+            </div>
+          ) : (
+            <div className="w-full grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-6 px-2 py-4">
+            {currentDocuments.map((document, index) => {
+              const docId = document.id ? document.id.toString() : '';
+
+              return (
+                <div key={docId} className="relative">
+                  <div
+                    className={`flex h-[130px] w-full flex-col rounded-2xl bg-[#F9F9F9] p-3 transition-all duration-200  dark:bg-[#383838] ${
+                      activeMenuId === docId || syncMenuState.docId === docId
+                        ? 'scale-[1.05]'
+                        : 'hover:scale-[1.05]'
+                    }`}
+                  >
+                      <div className="w-full flex-1">
+                        <div className="flex w-full items-center justify-between gap-2">
+                          <h3
+                            className="font-inter dark:text-bright-gray line-clamp-3 text-[13px] leading-[18px] font-semibold break-words text-[#18181B]"
+                            title={document.name}
+                          >
+                            {document.name}
+                          </h3>
+                          <div
+                            ref={getMenuRef(docId)}
+                            className="relative flex items-center justify-end"
+                          >
+                            {document.syncFrequency && (
+                              <DropdownMenu
+                                name={t('settings.sources.sync')}
+                                options={syncOptions}
+                                onSelect={(value: string) => {
+                                  handleManageSync(document, value);
+                                }}
+                                defaultValue={document.syncFrequency}
+                                icon={SyncIcon}
+                                isOpen={
+                                  syncMenuState.docId === docId &&
+                                  syncMenuState.isOpen
+                                }
+                                onOpenChange={(isOpen) => {
+                                  setSyncMenuState((prev) => ({
+                                    ...prev,
+                                    isOpen,
+                                    docId: isOpen ? docId : null,
+                                    document: isOpen ? document : null,
+                                  }));
+                                }}
+                                anchorRef={getMenuRef(docId)}
+                                position="bottom-left"
+                                offset={{ x: -8, y: 8 }}
+                                className="min-w-[120px]"
+                              />
+                            )}
+                            <button
+                              onClick={(e) => {
+                                e.stopPropagation();
+                                handleMenuClick(e, docId);
+                              }}
+                              className="inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E]"
+                              aria-label={t('settings.sources.menuAlt')}
+                              data-testid={`menu-button-${docId}`}
+                            >
+                              <img
+                                src={ThreeDots}
+                                alt={t('settings.sources.menuAlt')}
+                                className="opacity-60 hover:opacity-100"
+                              />
+                            </button>
+                          </div>
+                        </div>
+                      </div>
+
+                      <div className="flex flex-col items-start justify-start gap-1">
+                        <div className="flex items-center gap-2">
+                          <img
+                            src={CalendarIcon}
+                            alt=""
+                            className="w-[14px] h-[14px]"
+                          />
+                          <span className="font-inter text-[12px] leading-[18px] font-[500] text-[#848484] dark:text-[#848484]">
+                            {document.date ? formatDate(document.date) : ''}
+                          </span>
+                        </div>
+                        <div className="flex items-center gap-2">
+                          <img
+                            src={DiscIcon}
+                            alt=""
+                            className="w-[14px] h-[14px]"
+                          />
+                          <span className="font-inter text-[12px] leading-[18px] font-[500] text-[#848484] dark:text-[#848484]">
+                            {document.tokens
+                              ? formatTokens(+document.tokens)
+                              : ''}
+                          </span>
+                        </div>
+                      </div>
+                    </div>
+                    <ContextMenu
+                      isOpen={activeMenuId === docId}
+                      setIsOpen={(isOpen) => {
+                        setActiveMenuId(isOpen ? docId : null);
+                      }}
+                      options={getActionOptions(index, document)}
+                      anchorRef={getMenuRef(docId)}
+                      position="bottom-left"
+                      offset={{ x: -8, y: 8 }}
+                      className="z-50"
+                    />
+                  </div>
+                );
+              })}
+            </div>
+          )}
+        </div>
+      </div>
+
+      {currentDocuments.length > 0 && totalPages > 1 && (
+        <div className="mt-auto pt-4">
+          <Pagination
+            currentPage={currentPage}
+            totalPages={totalPages}
+            rowsPerPage={rowsPerPage}
+            onPageChange={(page) => {
+              setCurrentPage(page);
+              refreshDocs(undefined, page, rowsPerPage);
+            }}
+            onRowsPerPageChange={(rows) => {
+              setRowsPerPage(rows);
+              setCurrentPage(1);
+              refreshDocs(undefined, 1, rows);
+            }}
+          />
+        </div>
+      )}
+
+      {modalState === 'ACTIVE' && (
+        <Upload
+          receivedFile={[]}
+          setModalState={setModalState}
+          isOnboarding={isOnboarding}
+          renderTab={null}
+          close={() => setModalState('INACTIVE')}
+          onSuccessfulUpload={() =>
+            refreshDocs(undefined, currentPage, rowsPerPage)
+          }
+        />
+      )}
+
+      {deleteModalState === 'ACTIVE' && documentToDelete && (
+        <ConfirmationModal
+          message={t('settings.sources.deleteWarning', {
+            name: documentToDelete.document.name,
+          })}
+          modalState={deleteModalState}
+          setModalState={setDeleteModalState}
+          handleSubmit={handleConfirmedDelete}
+          handleCancel={() => {
+            setDeleteModalState('INACTIVE');
+            setDocumentToDelete(null);
+          }}
+          submitLabel={t('convTile.delete')}
+          variant="danger"
+        />
+      )}
+    </div>
+  );
+}

frontend/src/settings/index.tsx

@@ -21,7 +21,7 @@ import {
   setSourceDocs,
 } from '../preferences/preferenceSlice';
 import Analytics from './Analytics';
-import Documents from './Documents';
+import Sources from './Sources';
 import General from './General';
 import Logs from './Logs';
 import Tools from './Tools';
@@ -38,8 +38,8 @@ export default function Settings() {
 
   const getActiveTabFromPath = () => {
     const path = location.pathname;
-    if (path.includes('/settings/documents'))
-      return t('settings.documents.label');
+    if (path.includes('/settings/sources'))
+      return t('settings.sources.label');
     if (path.includes('/settings/analytics'))
       return t('settings.analytics.label');
     if (path.includes('/settings/logs')) return t('settings.logs.label');
@@ -53,8 +53,8 @@ export default function Settings() {
   const handleTabChange = (tab: string) => {
     setActiveTab(tab);
     if (tab === t('settings.general.label')) navigate('/settings');
-    else if (tab === t('settings.documents.label'))
-      navigate('/settings/documents');
+    else if (tab === t('settings.sources.label'))
+      navigate('/settings/sources');
     else if (tab === t('settings.analytics.label'))
       navigate('/settings/analytics');
     else if (tab === t('settings.logs.label')) navigate('/settings/logs');
@@ -113,9 +113,9 @@ export default function Settings() {
       <Routes>
         <Route index element={<General />} />
         <Route
-          path="documents"
+          path="sources"
           element={
-            <Documents
+            <Sources
               paginatedDocuments={paginatedDocuments}
               handleDeleteDocument={handleDeleteClick}
             />

frontend/src/utils/dateTimeUtils.ts

@@ -34,6 +34,20 @@ export function formatDate(dateString: string): string {
       day: 'numeric',
       year: 'numeric',
     });
+  } else if (
+    /^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(
+      dateString,
+    )
+  ) {
+    // Format: "Fri, 08 Jul 2025 06:00:00 GMT"
+    const dateTime = new Date(dateString);
+    return dateTime.toLocaleDateString('en-US', {
+      month: 'short',
+      day: 'numeric',
+      year: 'numeric',
+      hour: '2-digit',
+      minute: '2-digit',
+    });
   } else {
     return dateString;
   }