From 5923781484a02c8b84390a83eaa18503cad0237a Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Fri, 4 Apr 2025 03:29:01 +0530 Subject: [PATCH 01/19] (feat:attach) warning for error, progress, removal --- frontend/src/components/MessageInput.tsx | 173 +++++++++++------- .../src/conversation/conversationSlice.ts | 4 + 2 files changed, 108 insertions(+), 69 deletions(-) diff --git a/frontend/src/components/MessageInput.tsx b/frontend/src/components/MessageInput.tsx index 1e3576ee..62e6dea0 100644 --- a/frontend/src/components/MessageInput.tsx +++ b/frontend/src/components/MessageInput.tsx @@ -9,13 +9,16 @@ import SourceIcon from '../assets/source.svg'; import ToolIcon from '../assets/tool.svg'; import SpinnerDark from '../assets/spinner-dark.svg'; import Spinner from '../assets/spinner.svg'; +import ExitIcon from '../assets/exit.svg'; +import AlertIcon from '../assets/alert.svg'; import SourcesPopup from './SourcesPopup'; import ToolsPopup from './ToolsPopup'; import { selectSelectedDocs, selectToken } from '../preferences/preferenceSlice'; import { ActiveState } from '../models/misc'; import Upload from '../upload/Upload'; import ClipIcon from '../assets/clip.svg'; -import { setAttachments } from '../conversation/conversationSlice'; +import { setAttachments, removeAttachment } from '../conversation/conversationSlice'; + interface MessageInputProps { value: string; @@ -71,50 +74,59 @@ export default function MessageInput({ status: 'uploading' }; - setUploads(prev => [...prev, uploadState]); - const uploadIndex = uploads.length; - - xhr.upload.addEventListener('progress', (event) => { - if (event.lengthComputable) { - const progress = Math.round((event.loaded / event.total) * 100); - setUploads(prev => prev.map((upload, index) => - index === uploadIndex - ? { ...upload, progress } - : upload - )); - } - }); - - xhr.onload = () => { - if (xhr.status === 200) { - const response = JSON.parse(xhr.responseText); - console.log('File uploaded successfully:', response); - - if (response.task_id) { - setUploads(prev => prev.map((upload, index) => - index === uploadIndex - ? { ...upload, taskId: response.task_id, status: 'processing' } + setUploads(prev => { + const newUploads = [...prev, uploadState]; + const uploadIndex = newUploads.length - 1; + + xhr.upload.addEventListener('progress', (event) => { + if (event.lengthComputable) { + const progress = Math.round((event.loaded / event.total) * 100); + setUploads(current => current.map((upload, idx) => + idx === uploadIndex + ? { ...upload, progress } : upload )); } - } else { - setUploads(prev => prev.map((upload, index) => - index === uploadIndex + }); + + xhr.onload = () => { + if (xhr.status === 200) { + const response = JSON.parse(xhr.responseText); + console.log('File uploaded successfully:', response); + + if (response.task_id) { + setUploads(current => current.map((upload, idx) => + idx === uploadIndex + ? { + ...upload, + taskId: response.task_id, + status: 'processing', + progress: 10 + } + : upload + )); + } + } else { + setUploads(current => current.map((upload, idx) => + idx === uploadIndex + ? { ...upload, status: 'failed' } + : upload + )); + console.error('Error uploading file:', xhr.responseText); + } + }; + + xhr.onerror = () => { + setUploads(current => current.map((upload, idx) => + idx === uploadIndex ? { ...upload, status: 'failed' } : upload )); - console.error('Error uploading file:', xhr.responseText); - } - }; - - xhr.onerror = () => { - setUploads(prev => prev.map((upload, index) => - index === uploadIndex - ? { ...upload, status: 'failed' } - : upload - )); - console.error('Network error during file upload'); - }; + console.error('Network error during file upload'); + }; + + return newUploads; + }); xhr.open('POST', `${apiHost}${endpoints.USER.STORE_ATTACHMENT}`); xhr.setRequestHeader('Authorization', `Bearer ${token}`); @@ -233,45 +245,68 @@ export default function MessageInput({ {uploads.map((upload, index) => (
{upload.fileName} {upload.status === 'completed' && ( - ✓ + )} {upload.status === 'failed' && ( - ✗ + Upload failed )} - {(upload.status === 'uploading' || upload.status === 'processing') && ( -
- - - - -
- )} +{(upload.status === 'uploading' || upload.status === 'processing') && ( +
+ + {/* Background circle */} + + + +
+)}
))} diff --git a/frontend/src/conversation/conversationSlice.ts b/frontend/src/conversation/conversationSlice.ts index d38ba21b..fd07c707 100644 --- a/frontend/src/conversation/conversationSlice.ts +++ b/frontend/src/conversation/conversationSlice.ts @@ -289,6 +289,9 @@ export const conversationSlice = createSlice({ setAttachments: (state, action: PayloadAction<{ fileName: string; id: string }[]>) => { state.attachments = action.payload; }, + removeAttachment(state, action: PayloadAction) { + state.attachments = state.attachments?.filter(att => att.id !== action.payload); + }, }, extraReducers(builder) { builder @@ -323,5 +326,6 @@ export const { updateToolCalls, setConversation, setAttachments, + removeAttachment, } = conversationSlice.actions; export default conversationSlice.reducer; From e4945b41e9d90b062f15729e5768738b7670c0dd Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Sun, 6 Apr 2025 15:57:18 +0530 Subject: [PATCH 02/19] (feat:files) link attachment to openai_api --- application/llm/openai.py | 134 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/application/llm/openai.py b/application/llm/openai.py index 5e11a072..9da09c15 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -35,6 +35,15 @@ class OpenAILLM(BaseLLM): cleaned_messages.append( {"role": role, "content": item["text"]} ) + elif isinstance(item, dict): + content_parts = [] + if "text" in item: + content_parts.append({"type": "text", "text": item["text"]}) + elif "type" in item and item["type"] == "text" and "text" in item: + content_parts.append(item) + elif "type" in item and item["type"] == "file" and "file" in item: + content_parts.append(item) + cleaned_messages.append({"role": role, "content": content_parts}) elif "function_call" in item: tool_call = { "id": item["function_call"]["call_id"], @@ -133,6 +142,131 @@ class OpenAILLM(BaseLLM): def _supports_tools(self): return True + def prepare_messages_with_attachments(self, messages, attachments=None): + """ + Process attachments using OpenAI's file API for more efficient handling. + + Args: + messages (list): List of message dictionaries. + attachments (list): List of attachment dictionaries with content and metadata. + + Returns: + list: Messages formatted with file references for OpenAI API. + """ + if not attachments: + return messages + + prepared_messages = messages.copy() + + # Find the user message to attach file_id to the last one + user_message_index = None + for i in range(len(prepared_messages) - 1, -1, -1): + if prepared_messages[i].get("role") == "user": + user_message_index = i + break + + if user_message_index is None: + user_message = {"role": "user", "content": []} + prepared_messages.append(user_message) + user_message_index = len(prepared_messages) - 1 + + if isinstance(prepared_messages[user_message_index].get("content"), str): + text_content = prepared_messages[user_message_index]["content"] + prepared_messages[user_message_index]["content"] = [ + {"type": "text", "text": text_content} + ] + elif not isinstance(prepared_messages[user_message_index].get("content"), list): + prepared_messages[user_message_index]["content"] = [] + + for attachment in attachments: + # Upload the file to OpenAI + try: + file_id = self._upload_file_to_openai(attachment) + + prepared_messages[user_message_index]["content"].append({ + "type": "file", + "file": {"file_id": file_id} + }) + except Exception as e: + import logging + logging.error(f"Error uploading attachment to OpenAI: {e}") + if 'content' in attachment: + prepared_messages[user_message_index]["content"].append({ + "type": "text", + "text": f"File content:\n\n{attachment['content']}" + }) + + return prepared_messages + + def _upload_file_to_openai(self, attachment): + """ + Upload a file to OpenAI and return the file_id. + + Args: + attachment (dict): Attachment dictionary with path and metadata. + Expected keys: + - path: Path to the file + - id: Optional MongoDB ID for caching + + Returns: + str: OpenAI file_id for the uploaded file. + """ + import os + import mimetypes + + # Check if we already have the file_id cached + if 'openai_file_id' in attachment: + return attachment['openai_file_id'] + + file_path = attachment.get('path') + if not file_path: + raise ValueError("No file path provided in attachment") + + # Make path absolute if it's relative + if not os.path.isabs(file_path): + current_dir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + file_path = os.path.join(current_dir,"application", file_path) + + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + mime_type = attachment.get('mime_type') + if not mime_type: + mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' + + supported_mime_types = ['application/pdf', 'image/png', 'image/jpeg', 'image/gif'] + if mime_type not in supported_mime_types: + import logging + logging.warning(f"MIME type {mime_type} not supported by OpenAI for file uploads. Falling back to text.") + raise ValueError(f"Unsupported MIME type: {mime_type}") + + try: + with open(file_path, 'rb') as file: + response = self.client.files.create( + file=file, + purpose="assistants" + ) + + file_id = response.id + + from application.core.mongo_db import MongoDB + mongo = MongoDB.get_client() + db = mongo["docsgpt"] + attachments_collection = db["attachments"] + if '_id' in attachment: + attachments_collection.update_one( + {"_id": attachment['_id']}, + {"$set": {"openai_file_id": file_id}} + ) + + return file_id + except Exception as e: + import logging + logging.error(f"Error uploading file to OpenAI: {e}") + raise + class AzureOpenAILLM(OpenAILLM): From a37bd7695038685b8a81036652488267d8ac4fd4 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Sun, 6 Apr 2025 16:01:57 +0530 Subject: [PATCH 03/19] (feat:storeAttach) store in inputs, raise errors from worker --- application/api/user/routes.py | 25 ++++++++++++--------- application/worker.py | 41 +++++++++++++++------------------- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 8f374aa7..91b028d5 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -2506,23 +2506,26 @@ class StoreAttachment(Resource): user = secure_filename(decoded_token.get("sub")) try: + attachment_id = ObjectId() original_filename = secure_filename(file.filename) - folder_name = original_filename - save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, "attachments",folder_name) + + save_dir = os.path.join( + current_dir, + settings.UPLOAD_FOLDER, + user, + "attachments", + str(attachment_id) + ) os.makedirs(save_dir, exist_ok=True) - # Create directory structure: user/attachments/filename/ + file_path = os.path.join(save_dir, original_filename) - # Handle filename conflicts - if os.path.exists(file_path): - name_parts = os.path.splitext(original_filename) - timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") - new_filename = f"{name_parts[0]}_{timestamp}{name_parts[1]}" - file_path = os.path.join(save_dir, new_filename) - original_filename = new_filename file.save(file_path) - file_info = {"folder": folder_name, "filename": original_filename} + file_info = { + "filename": original_filename, + "attachment_id": str(attachment_id) + } current_app.logger.info(f"Saved file: {file_path}") # Start async task to process single file diff --git a/application/worker.py b/application/worker.py index 23ff0422..0437c38a 100755 --- a/application/worker.py +++ b/application/worker.py @@ -334,28 +334,23 @@ def attachment_worker(self, directory, file_info, user): db = mongo["docsgpt"] attachments_collection = db["attachments"] - job_name = file_info["folder"] - logging.info(f"Processing attachment: {job_name}", extra={"user": user, "job": job_name}) + filename = file_info["filename"] + attachment_id = file_info["attachment_id"] + + logging.info(f"Processing attachment: {attachment_id}/{filename}", extra={"user": user}) self.update_state(state="PROGRESS", meta={"current": 10}) - folder_name = file_info["folder"] - filename = file_info["filename"] - file_path = os.path.join(directory, filename) - - logging.info(f"Processing file: {file_path}", extra={"user": user, "job": job_name}) - if not os.path.exists(file_path): - logging.warning(f"File not found: {file_path}", extra={"user": user, "job": job_name}) - return {"error": "File not found"} + logging.warning(f"File not found: {file_path}", extra={"user": user}) + raise FileNotFoundError(f"File not found: {file_path}") try: reader = SimpleDirectoryReader( input_files=[file_path] ) - documents = reader.load_data() self.update_state(state="PROGRESS", meta={"current": 50}) @@ -364,33 +359,33 @@ def attachment_worker(self, directory, file_info, user): content = documents[0].text token_count = num_tokens_from_string(content) - file_path_relative = f"{user}/attachments/{folder_name}/{filename}" + file_path_relative = f"{settings.UPLOAD_FOLDER}/{user}/attachments/{attachment_id}/{filename}" - attachment_id = attachments_collection.insert_one({ + doc_id = ObjectId(attachment_id) + attachments_collection.insert_one({ + "_id": doc_id, "user": user, "path": file_path_relative, "content": content, "token_count": token_count, "date": datetime.datetime.now(), - }).inserted_id + }) logging.info(f"Stored attachment with ID: {attachment_id}", - extra={"user": user, "job": job_name}) + extra={"user": user}) self.update_state(state="PROGRESS", meta={"current": 100}) return { - "attachment_id": str(attachment_id), "filename": filename, - "folder": folder_name, "path": file_path_relative, - "token_count": token_count + "token_count": token_count, + "attachment_id": attachment_id } else: logging.warning("No content was extracted from the file", - extra={"user": user, "job": job_name}) - return {"error": "No content was extracted from the file"} + extra={"user": user}) + raise ValueError("No content was extracted from the file") except Exception as e: - logging.error(f"Error processing file {filename}: {e}", - extra={"user": user, "job": job_name}, exc_info=True) - return {"error": f"Error processing file: {str(e)}"} + logging.error(f"Error processing file {filename}: {e}", extra={"user": user}, exc_info=True) + raise \ No newline at end of file From 244c9b96a2dbafda7751de3f6d8c67515fc08ca6 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Sun, 6 Apr 2025 16:02:30 +0530 Subject: [PATCH 04/19] (fix:attach) pass attachment docs as it is --- application/api/answer/routes.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 7f61880d..97fce28c 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -831,12 +831,7 @@ def get_attachments_content(attachment_ids, user): }) if attachment_doc: - attachments.append({ - "id": str(attachment_doc["_id"]), - "content": attachment_doc["content"], - "token_count": attachment_doc.get("token_count", 0), - "path": attachment_doc.get("path", "") - }) + attachments.append(attachment_doc) except Exception as e: logger.error(f"Error retrieving attachment {attachment_id}: {e}") From 1f3d1cc73ea366a4f10a397328401941df35a254 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Mon, 7 Apr 2025 20:15:11 +0530 Subject: [PATCH 05/19] (feat:attach) handle unsupported attachments --- application/agents/llm_handler.py | 52 ++++++++++++++++++++++++++----- application/llm/base.py | 9 ++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/application/agents/llm_handler.py b/application/agents/llm_handler.py index 21d972d9..80acc2bb 100644 --- a/application/agents/llm_handler.py +++ b/application/agents/llm_handler.py @@ -15,7 +15,7 @@ class LLMHandler(ABC): @abstractmethod def handle_response(self, agent, resp, tools_dict, messages, attachments=None, **kwargs): pass - + def prepare_messages_with_attachments(self, agent, messages, attachments=None): """ Prepare messages with attachment content if available. @@ -33,15 +33,53 @@ class LLMHandler(ABC): logger.info(f"Preparing messages with {len(attachments)} attachments") - # Check if the LLM has its own custom attachment handling implementation - if hasattr(agent.llm, "prepare_messages_with_attachments") and agent.llm.__class__.__name__ != "BaseLLM": - logger.info(f"Using {agent.llm.__class__.__name__}'s own prepare_messages_with_attachments method") - return agent.llm.prepare_messages_with_attachments(messages, attachments) + supported_types = agent.llm.get_supported_attachment_types() - # Otherwise, append attachment content to the system prompt + supported_attachments = [] + unsupported_attachments = [] + + for attachment in attachments: + mime_type = attachment.get('mime_type') + if not mime_type: + import mimetypes + file_path = attachment.get('path') + if file_path: + mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' + else: + unsupported_attachments.append(attachment) + continue + + if mime_type in supported_types: + supported_attachments.append(attachment) + else: + unsupported_attachments.append(attachment) + + # Process supported attachments with the LLM's custom method + prepared_messages = messages + if supported_attachments: + logger.info(f"Processing {len(supported_attachments)} supported attachments with {agent.llm.__class__.__name__}'s method") + prepared_messages = agent.llm.prepare_messages_with_attachments(messages, supported_attachments) + + # Process unsupported attachments with the default method + if unsupported_attachments: + logger.info(f"Processing {len(unsupported_attachments)} unsupported attachments with default method") + prepared_messages = self._append_attachment_content_to_system(prepared_messages, unsupported_attachments) + + return prepared_messages + + def _append_attachment_content_to_system(self, messages, attachments): + """ + Default method to append attachment content to the system prompt. + + Args: + messages (list): List of message dictionaries. + attachments (list): List of attachment dictionaries with content. + + Returns: + list: Messages with attachment context added to the system prompt. + """ prepared_messages = messages.copy() - # Build attachment content string attachment_texts = [] for attachment in attachments: logger.info(f"Adding attachment {attachment.get('id')} to context") diff --git a/application/llm/base.py b/application/llm/base.py index 0fce208c..0607159d 100644 --- a/application/llm/base.py +++ b/application/llm/base.py @@ -55,3 +55,12 @@ class BaseLLM(ABC): def _supports_tools(self): raise NotImplementedError("Subclass must implement _supports_tools method") + + def get_supported_attachment_types(self): + """ + Return a list of MIME types supported by this LLM for file uploads. + + Returns: + list: List of supported MIME types + """ + return [] # Default: no attachments supported From 0c1138179b811a2a5e117e5aa24546c66b43622b Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Mon, 7 Apr 2025 20:16:03 +0530 Subject: [PATCH 06/19] (feat:attch) store file mime type --- application/worker.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/application/worker.py b/application/worker.py index 0437c38a..bbd422ac 100755 --- a/application/worker.py +++ b/application/worker.py @@ -328,6 +328,7 @@ def attachment_worker(self, directory, file_info, user): """ import datetime import os + import mimetypes from application.utils import num_tokens_from_string mongo = MongoDB.get_client() @@ -361,6 +362,8 @@ def attachment_worker(self, directory, file_info, user): file_path_relative = f"{settings.UPLOAD_FOLDER}/{user}/attachments/{attachment_id}/{filename}" + mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' + doc_id = ObjectId(attachment_id) attachments_collection.insert_one({ "_id": doc_id, @@ -368,6 +371,7 @@ def attachment_worker(self, directory, file_info, user): "path": file_path_relative, "content": content, "token_count": token_count, + "mime_type": mime_type, "date": datetime.datetime.now(), }) @@ -380,7 +384,8 @@ def attachment_worker(self, directory, file_info, user): "filename": filename, "path": file_path_relative, "token_count": token_count, - "attachment_id": attachment_id + "attachment_id": attachment_id, + "mime_type": mime_type } else: logging.warning("No content was extracted from the file", @@ -388,4 +393,4 @@ def attachment_worker(self, directory, file_info, user): raise ValueError("No content was extracted from the file") except Exception as e: logging.error(f"Error processing file {filename}: {e}", extra={"user": user}, exc_info=True) - raise \ No newline at end of file + raise From 5421bc13861cfbe8f7a7c66a79183d2fec8cc0dd Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Tue, 8 Apr 2025 15:51:37 +0530 Subject: [PATCH 07/19] (feat:attach) extend support for imgs --- application/llm/openai.py | 114 ++++++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 29 deletions(-) diff --git a/application/llm/openai.py b/application/llm/openai.py index 9da09c15..ad0a0d56 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -1,4 +1,8 @@ import json +import base64 +import os +import mimetypes +import logging from application.core.settings import settings from application.llm.base import BaseLLM @@ -142,6 +146,22 @@ class OpenAILLM(BaseLLM): def _supports_tools(self): return True + def get_supported_attachment_types(self): + """ + Return a list of MIME types supported by OpenAI for file uploads. + + Returns: + list: List of supported MIME types + """ + return [ + 'application/pdf', + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/webp', + 'image/gif' + ] + def prepare_messages_with_attachments(self, messages, attachments=None): """ Process attachments using OpenAI's file API for more efficient handling. @@ -179,26 +199,74 @@ class OpenAILLM(BaseLLM): prepared_messages[user_message_index]["content"] = [] for attachment in attachments: - # Upload the file to OpenAI - try: - file_id = self._upload_file_to_openai(attachment) - - prepared_messages[user_message_index]["content"].append({ - "type": "file", - "file": {"file_id": file_id} - }) - except Exception as e: - import logging - logging.error(f"Error uploading attachment to OpenAI: {e}") - if 'content' in attachment: + mime_type = attachment.get('mime_type') + if not mime_type: + file_path = attachment.get('path') + if file_path: + mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' + + if mime_type and mime_type.startswith('image/'): + try: + base64_image = self._get_base64_image(attachment) prepared_messages[user_message_index]["content"].append({ - "type": "text", - "text": f"File content:\n\n{attachment['content']}" + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{base64_image}" + } }) + except Exception as e: + logging.error(f"Error processing image attachment: {e}") + if 'content' in attachment: + prepared_messages[user_message_index]["content"].append({ + "type": "text", + "text": f"[Image could not be processed: {attachment.get('path', 'unknown')}]" + }) + # Handle PDFs using the file API + elif mime_type == 'application/pdf': + try: + file_id = self._upload_file_to_openai(attachment) + + prepared_messages[user_message_index]["content"].append({ + "type": "file", + "file": {"file_id": file_id} + }) + except Exception as e: + logging.error(f"Error uploading PDF to OpenAI: {e}") + if 'content' in attachment: + prepared_messages[user_message_index]["content"].append({ + "type": "text", + "text": f"File content:\n\n{attachment['content']}" + }) return prepared_messages - def _upload_file_to_openai(self, attachment): + def _get_base64_image(self, attachment): + """ + Convert an image file to base64 encoding. + + Args: + attachment (dict): Attachment dictionary with path and metadata. + + Returns: + str: Base64-encoded image data. + """ + file_path = attachment.get('path') + if not file_path: + raise ValueError("No file path provided in attachment") + + if not os.path.isabs(file_path): + current_dir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + file_path = os.path.join(current_dir, "application", file_path) + + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + with open(file_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode('utf-8') + + def _upload_file_to_openai(self, attachment): ##pdfs """ Upload a file to OpenAI and return the file_id. @@ -212,9 +280,8 @@ class OpenAILLM(BaseLLM): str: OpenAI file_id for the uploaded file. """ import os - import mimetypes + import logging - # Check if we already have the file_id cached if 'openai_file_id' in attachment: return attachment['openai_file_id'] @@ -222,7 +289,6 @@ class OpenAILLM(BaseLLM): if not file_path: raise ValueError("No file path provided in attachment") - # Make path absolute if it's relative if not os.path.isabs(file_path): current_dir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -231,16 +297,7 @@ class OpenAILLM(BaseLLM): if not os.path.exists(file_path): raise FileNotFoundError(f"File not found: {file_path}") - - mime_type = attachment.get('mime_type') - if not mime_type: - mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' - - supported_mime_types = ['application/pdf', 'image/png', 'image/jpeg', 'image/gif'] - if mime_type not in supported_mime_types: - import logging - logging.warning(f"MIME type {mime_type} not supported by OpenAI for file uploads. Falling back to text.") - raise ValueError(f"Unsupported MIME type: {mime_type}") + try: with open(file_path, 'rb') as file: @@ -263,7 +320,6 @@ class OpenAILLM(BaseLLM): return file_id except Exception as e: - import logging logging.error(f"Error uploading file to OpenAI: {e}") raise From cd7bbb45c3bd012d7b7c59657164a5242fda8bd9 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Tue, 8 Apr 2025 17:51:18 +0530 Subject: [PATCH 08/19] (fix:popups) minor hover --- frontend/src/components/SourcesPopup.tsx | 2 +- frontend/src/components/ToolsPopup.tsx | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/SourcesPopup.tsx b/frontend/src/components/SourcesPopup.tsx index 9f8ce8ce..088b3eb1 100644 --- a/frontend/src/components/SourcesPopup.tsx +++ b/frontend/src/components/SourcesPopup.tsx @@ -207,7 +207,7 @@ export default function SourcesPopup({ )} -
+ ); -} \ No newline at end of file +} From dd9ea46e58c2de59ebf8587040a04d3205df7f4d Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Thu, 10 Apr 2025 01:29:01 +0530 Subject: [PATCH 09/19] (feat:attach) strategy specific to google genai --- application/llm/google_ai.py | 148 ++++++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 1 deletion(-) diff --git a/application/llm/google_ai.py b/application/llm/google_ai.py index 5e33550c..180ed43b 100644 --- a/application/llm/google_ai.py +++ b/application/llm/google_ai.py @@ -1,5 +1,8 @@ from google import genai from google.genai import types +import os +import logging +import mimetypes from application.llm.base import BaseLLM @@ -9,6 +12,141 @@ class GoogleLLM(BaseLLM): super().__init__(*args, **kwargs) self.api_key = api_key self.user_api_key = user_api_key + self.client = genai.Client(api_key=self.api_key) + + def get_supported_attachment_types(self): + """ + Return a list of MIME types supported by Google Gemini for file uploads. + + Returns: + list: List of supported MIME types + """ + return [ + 'application/pdf', + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/webp', + 'image/gif' + ] + + def prepare_messages_with_attachments(self, messages, attachments=None): + """ + Process attachments using Google AI's file API for more efficient handling. + + Args: + messages (list): List of message dictionaries. + attachments (list): List of attachment dictionaries with content and metadata. + + Returns: + list: Messages formatted with file references for Google AI API. + """ + + if not attachments: + return messages + + prepared_messages = messages.copy() + + # Find the user message to attach files to the last one + user_message_index = None + for i in range(len(prepared_messages) - 1, -1, -1): + if prepared_messages[i].get("role") == "user": + user_message_index = i + break + + + if user_message_index is None: + user_message = {"role": "user", "content": []} + prepared_messages.append(user_message) + user_message_index = len(prepared_messages) - 1 + + if isinstance(prepared_messages[user_message_index].get("content"), str): + text_content = prepared_messages[user_message_index]["content"] + prepared_messages[user_message_index]["content"] = [ + {"type": "text", "text": text_content} + ] + elif not isinstance(prepared_messages[user_message_index].get("content"), list): + prepared_messages[user_message_index]["content"] = [] + + file_uris = [] + for attachment in attachments: + mime_type = attachment.get('mime_type') + if not mime_type: + file_path = attachment.get('path') + if file_path: + mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' + + if mime_type in self.get_supported_attachment_types(): + try: + file_uri = self._upload_file_to_google(attachment) + logging.info(f"GoogleLLM: Successfully uploaded file, got URI: {file_uri}") + file_uris.append((file_uri, mime_type)) + except Exception as e: + logging.error(f"GoogleLLM: Error uploading file: {e}") + if 'content' in attachment: + prepared_messages[user_message_index]["content"].append({ + "type": "text", + "text": f"[File could not be processed: {attachment.get('path', 'unknown')}]" + }) + + if file_uris: + logging.info(f"GoogleLLM: Adding {len(file_uris)} file URIs to message") + prepared_messages[user_message_index]["content"].append({ + "type": "file_uris", + "file_uris": file_uris + }) + + return prepared_messages + + def _upload_file_to_google(self, attachment): + """ + Upload a file to Google AI and return the file URI. + + Args: + attachment (dict): Attachment dictionary with path and metadata. + + Returns: + str: Google AI file URI for the uploaded file. + """ + if 'google_file_uri' in attachment: + return attachment['google_file_uri'] + + file_path = attachment.get('path') + if not file_path: + raise ValueError("No file path provided in attachment") + + if not os.path.isabs(file_path): + current_dir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + file_path = os.path.join(current_dir, "application", file_path) + + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + mime_type = attachment.get('mime_type') + if not mime_type: + mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' + + try: + response = self.client.files.upload(file=file_path) + + file_uri = response.uri + + from application.core.mongo_db import MongoDB + mongo = MongoDB.get_client() + db = mongo["docsgpt"] + attachments_collection = db["attachments"] + if '_id' in attachment: + attachments_collection.update_one( + {"_id": attachment['_id']}, + {"$set": {"google_file_uri": file_uri}} + ) + + return file_uri + except Exception as e: + logging.error(f"Error uploading file to Google AI: {e}") + raise def _clean_messages_google(self, messages): cleaned_messages = [] @@ -26,7 +164,7 @@ class GoogleLLM(BaseLLM): elif isinstance(content, list): for item in content: if "text" in item: - parts.append(types.Part.from_text(item["text"])) + parts.append(types.Part.from_text(text=item["text"])) elif "function_call" in item: parts.append( types.Part.from_function_call( @@ -41,6 +179,14 @@ class GoogleLLM(BaseLLM): response=item["function_response"]["response"], ) ) + elif "type" in item and item["type"] == "file_uris": + for file_uri, mime_type in item["file_uris"]: + parts.append( + types.Part.from_uri( + file_uri=file_uri, + mime_type=mime_type + ) + ) else: raise ValueError( f"Unexpected content dictionary format:{item}" From 6cb4577e1b79522dce99dd85b392acb09f26ae7a Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Thu, 10 Apr 2025 01:43:46 +0530 Subject: [PATCH 10/19] (feat:input) hotkey for sources open --- frontend/src/components/MessageInput.tsx | 30 ++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/MessageInput.tsx b/frontend/src/components/MessageInput.tsx index 62e6dea0..612cbaf2 100644 --- a/frontend/src/components/MessageInput.tsx +++ b/frontend/src/components/MessageInput.tsx @@ -4,6 +4,7 @@ import { useDarkTheme } from '../hooks'; import { useSelector, useDispatch } from 'react-redux'; import userService from '../api/services/userService'; import endpoints from '../api/endpoints'; +import { getOS, isTouchDevice } from '../utils/browserUtils'; import PaperPlane from '../assets/paper_plane.svg'; import SourceIcon from '../assets/source.svg'; import ToolIcon from '../assets/tool.svg'; @@ -57,6 +58,26 @@ export default function MessageInput({ const dispatch = useDispatch(); + const browserOS = getOS(); + const isTouch = isTouchDevice(); + + useEffect(() => { + const handleKeyDown = (event: KeyboardEvent) => { + if ( + ((browserOS === 'win' || browserOS === 'linux') && event.ctrlKey && event.key === 'k') || + (browserOS === 'mac' && event.metaKey && event.key === 'k') + ) { + event.preventDefault(); + setIsSourcesPopupOpen(!isSourcesPopupOpen); + } + }; + + document.addEventListener('keydown', handleKeyDown); + return () => { + document.removeEventListener('keydown', handleKeyDown); + }; + }, [browserOS]); + const handleFileAttachment = (e: React.ChangeEvent) => { if (!e.target.files || e.target.files.length === 0) return; @@ -333,15 +354,20 @@ export default function MessageInput({
)} - {upload.status === 'failed' && ( + {attachment.status === 'failed' && ( Upload failed )} -{(upload.status === 'uploading' || upload.status === 'processing') && ( -
- - {/* Background circle */} - - - -
-)} + {(attachment.status === 'uploading' || attachment.status === 'processing') && ( +
+ + {/* Background circle */} + + + +
+ )}
))}
diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx index ec244087..a241b2d3 100644 --- a/frontend/src/conversation/ConversationBubble.tsx +++ b/frontend/src/conversation/ConversationBubble.tsx @@ -57,7 +57,6 @@ const ConversationBubble = forwardRef< updated?: boolean, index?: number, ) => void; - attachments?: { fileName: string; id: string }[]; } >(function ConversationBubble( { @@ -72,7 +71,6 @@ const ConversationBubble = forwardRef< retryBtn, questionNumber, handleUpdatedQuestionSubmission, - attachments, }, ref, ) { @@ -99,36 +97,6 @@ const ConversationBubble = forwardRef< handleUpdatedQuestionSubmission?.(editInputBox, true, questionNumber); }; let bubble; - const renderAttachments = () => { - if (!attachments || attachments.length === 0) return null; - - return ( -
- {attachments.map((attachment, index) => ( -
- - - - {attachment.fileName} -
- ))} -
- ); - }; if (type === 'QUESTION') { bubble = (
{message}
- {renderAttachments()} )} - + {attachment.status === 'failed' && ( )} - + {(attachment.status === 'uploading' || attachment.status === 'processing') && (
@@ -328,6 +329,7 @@ export default function MessageInput({ ref={sourceButtonRef} className="flex items-center px-2 xs:px-3 py-1 xs:py-1.5 rounded-[32px] border border-[#AAAAAA] dark:border-purple-taupe hover:bg-gray-100 dark:hover:bg-[#2C2E3C] transition-colors max-w-[130px] sm:max-w-[150px]" onClick={() => setIsSourcesPopupOpen(!isSourcesPopupOpen)} + title={selectedDocs ? selectedDocs.name : t('conversation.sources.title')} > Sources From 02934452d693c1cf412f5a13e3a80a77bd9e84ae Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 14 Apr 2025 23:48:17 +0100 Subject: [PATCH 18/19] fix: remove comment --- application/agents/llm_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/application/agents/llm_handler.py b/application/agents/llm_handler.py index b9e7ba37..7fe794f8 100644 --- a/application/agents/llm_handler.py +++ b/application/agents/llm_handler.py @@ -164,7 +164,6 @@ class OpenAILLMHandler(LLMHandler): while True: tool_calls = {} for chunk in resp: - logger.info(f"Chunk: {chunk}") if isinstance(chunk, str) and len(chunk) > 0: yield chunk continue From ad610d2f90e682ad9c8bdefdd955b6270f92c135 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 14 Apr 2025 23:49:40 +0100 Subject: [PATCH 19/19] fix: lint ruff --- application/llm/google_ai.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/application/llm/google_ai.py b/application/llm/google_ai.py index 8ca5a4a0..c049eaa2 100644 --- a/application/llm/google_ai.py +++ b/application/llm/google_ai.py @@ -308,8 +308,6 @@ class GoogleLLM(BaseLLM): config=config, ) - # Track if we've seen any function calls - function_call_seen = False for chunk in response: if hasattr(chunk, "candidates") and chunk.candidates: @@ -317,7 +315,6 @@ class GoogleLLM(BaseLLM): if candidate.content and candidate.content.parts: for part in candidate.content.parts: if part.function_call: - function_call_seen = True yield part elif part.text: yield part.text