diff --git a/application/llm/openai.py b/application/llm/openai.py index 9da09c15..ad0a0d56 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -1,4 +1,8 @@ import json +import base64 +import os +import mimetypes +import logging from application.core.settings import settings from application.llm.base import BaseLLM @@ -142,6 +146,22 @@ class OpenAILLM(BaseLLM): def _supports_tools(self): return True + def get_supported_attachment_types(self): + """ + Return a list of MIME types supported by OpenAI for file uploads. + + Returns: + list: List of supported MIME types + """ + return [ + 'application/pdf', + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/webp', + 'image/gif' + ] + def prepare_messages_with_attachments(self, messages, attachments=None): """ Process attachments using OpenAI's file API for more efficient handling. @@ -179,26 +199,74 @@ class OpenAILLM(BaseLLM): prepared_messages[user_message_index]["content"] = [] for attachment in attachments: - # Upload the file to OpenAI - try: - file_id = self._upload_file_to_openai(attachment) - - prepared_messages[user_message_index]["content"].append({ - "type": "file", - "file": {"file_id": file_id} - }) - except Exception as e: - import logging - logging.error(f"Error uploading attachment to OpenAI: {e}") - if 'content' in attachment: + mime_type = attachment.get('mime_type') + if not mime_type: + file_path = attachment.get('path') + if file_path: + mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' + + if mime_type and mime_type.startswith('image/'): + try: + base64_image = self._get_base64_image(attachment) prepared_messages[user_message_index]["content"].append({ - "type": "text", - "text": f"File content:\n\n{attachment['content']}" + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{base64_image}" + } }) + except Exception as e: + logging.error(f"Error processing image attachment: {e}") + if 'content' in attachment: + prepared_messages[user_message_index]["content"].append({ + "type": "text", + "text": f"[Image could not be processed: {attachment.get('path', 'unknown')}]" + }) + # Handle PDFs using the file API + elif mime_type == 'application/pdf': + try: + file_id = self._upload_file_to_openai(attachment) + + prepared_messages[user_message_index]["content"].append({ + "type": "file", + "file": {"file_id": file_id} + }) + except Exception as e: + logging.error(f"Error uploading PDF to OpenAI: {e}") + if 'content' in attachment: + prepared_messages[user_message_index]["content"].append({ + "type": "text", + "text": f"File content:\n\n{attachment['content']}" + }) return prepared_messages - def _upload_file_to_openai(self, attachment): + def _get_base64_image(self, attachment): + """ + Convert an image file to base64 encoding. + + Args: + attachment (dict): Attachment dictionary with path and metadata. + + Returns: + str: Base64-encoded image data. + """ + file_path = attachment.get('path') + if not file_path: + raise ValueError("No file path provided in attachment") + + if not os.path.isabs(file_path): + current_dir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + file_path = os.path.join(current_dir, "application", file_path) + + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + with open(file_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode('utf-8') + + def _upload_file_to_openai(self, attachment): ##pdfs """ Upload a file to OpenAI and return the file_id. @@ -212,9 +280,8 @@ class OpenAILLM(BaseLLM): str: OpenAI file_id for the uploaded file. """ import os - import mimetypes + import logging - # Check if we already have the file_id cached if 'openai_file_id' in attachment: return attachment['openai_file_id'] @@ -222,7 +289,6 @@ class OpenAILLM(BaseLLM): if not file_path: raise ValueError("No file path provided in attachment") - # Make path absolute if it's relative if not os.path.isabs(file_path): current_dir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -231,16 +297,7 @@ class OpenAILLM(BaseLLM): if not os.path.exists(file_path): raise FileNotFoundError(f"File not found: {file_path}") - - mime_type = attachment.get('mime_type') - if not mime_type: - mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream' - - supported_mime_types = ['application/pdf', 'image/png', 'image/jpeg', 'image/gif'] - if mime_type not in supported_mime_types: - import logging - logging.warning(f"MIME type {mime_type} not supported by OpenAI for file uploads. Falling back to text.") - raise ValueError(f"Unsupported MIME type: {mime_type}") + try: with open(file_path, 'rb') as file: @@ -263,7 +320,6 @@ class OpenAILLM(BaseLLM): return file_id except Exception as e: - import logging logging.error(f"Error uploading file to OpenAI: {e}") raise