(feat:attach) extend support for imgs

application/llm/openai.py

@@ -1,4 +1,8 @@
 import json
+import base64
+import os
+import mimetypes
+import logging
 
 from application.core.settings import settings
 from application.llm.base import BaseLLM
@@ -142,6 +146,22 @@ class OpenAILLM(BaseLLM):
     def _supports_tools(self):
         return True
 
+    def get_supported_attachment_types(self):
+        """
+        Return a list of MIME types supported by OpenAI for file uploads.
+        
+        Returns:
+            list: List of supported MIME types
+        """
+        return [
+            'application/pdf',
+            'image/png',
+            'image/jpeg',
+            'image/jpg',
+            'image/webp',
+            'image/gif'
+        ]
+    
     def prepare_messages_with_attachments(self, messages, attachments=None):
         """
         Process attachments using OpenAI's file API for more efficient handling.
@@ -179,26 +199,74 @@ class OpenAILLM(BaseLLM):
             prepared_messages[user_message_index]["content"] = []
         
         for attachment in attachments:
-            # Upload the file to OpenAI
-            try:
-                file_id = self._upload_file_to_openai(attachment)
-                
-                prepared_messages[user_message_index]["content"].append({
-                    "type": "file",
-                    "file": {"file_id": file_id}
-                })
-            except Exception as e:
-                import logging
-                logging.error(f"Error uploading attachment to OpenAI: {e}")
-                if 'content' in attachment:
+            mime_type = attachment.get('mime_type')
+            if not mime_type:
+                file_path = attachment.get('path')
+                if file_path:
+                    mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream'
+            
+            if mime_type and mime_type.startswith('image/'):
+                try:
+                    base64_image = self._get_base64_image(attachment)
                     prepared_messages[user_message_index]["content"].append({
-                        "type": "text", 
-                        "text": f"File content:\n\n{attachment['content']}"
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:{mime_type};base64,{base64_image}"
+                        }
                     })
+                except Exception as e:
+                    logging.error(f"Error processing image attachment: {e}")
+                    if 'content' in attachment:
+                        prepared_messages[user_message_index]["content"].append({
+                            "type": "text", 
+                            "text": f"[Image could not be processed: {attachment.get('path', 'unknown')}]"
+                        })
+            # Handle PDFs using the file API
+            elif mime_type == 'application/pdf':
+                try:
+                    file_id = self._upload_file_to_openai(attachment)
+                    
+                    prepared_messages[user_message_index]["content"].append({
+                        "type": "file",
+                        "file": {"file_id": file_id}
+                    })
+                except Exception as e:
+                    logging.error(f"Error uploading PDF to OpenAI: {e}")
+                    if 'content' in attachment:
+                        prepared_messages[user_message_index]["content"].append({
+                            "type": "text", 
+                            "text": f"File content:\n\n{attachment['content']}"
+                        })
         
         return prepared_messages
 
-    def _upload_file_to_openai(self, attachment):
+    def _get_base64_image(self, attachment):
+        """
+        Convert an image file to base64 encoding.
+        
+        Args:
+            attachment (dict): Attachment dictionary with path and metadata.
+            
+        Returns:
+            str: Base64-encoded image data.
+        """
+        file_path = attachment.get('path')
+        if not file_path:
+            raise ValueError("No file path provided in attachment")
+        
+        if not os.path.isabs(file_path):
+            current_dir = os.path.dirname(
+                os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+            )
+            file_path = os.path.join(current_dir, "application", file_path)
+        
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+        
+        with open(file_path, "rb") as image_file:
+            return base64.b64encode(image_file.read()).decode('utf-8')
+
+    def _upload_file_to_openai(self, attachment): ##pdfs
         """
         Upload a file to OpenAI and return the file_id.
         
@@ -212,9 +280,8 @@ class OpenAILLM(BaseLLM):
             str: OpenAI file_id for the uploaded file.
         """
         import os
-        import mimetypes
+        import logging
         
-        # Check if we already have the file_id cached
         if 'openai_file_id' in attachment:
             return attachment['openai_file_id']
         
@@ -222,7 +289,6 @@ class OpenAILLM(BaseLLM):
         if not file_path:
             raise ValueError("No file path provided in attachment")
         
-        # Make path absolute if it's relative
         if not os.path.isabs(file_path):
             current_dir = os.path.dirname(
                 os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -231,16 +297,7 @@ class OpenAILLM(BaseLLM):
         
         if not os.path.exists(file_path):
             raise FileNotFoundError(f"File not found: {file_path}")
-        
-        mime_type = attachment.get('mime_type')
-        if not mime_type:
-            mime_type = mimetypes.guess_type(file_path)[0] or 'application/octet-stream'
-        
-        supported_mime_types = ['application/pdf', 'image/png', 'image/jpeg', 'image/gif']
-        if mime_type not in supported_mime_types:
-            import logging
-            logging.warning(f"MIME type {mime_type} not supported by OpenAI for file uploads. Falling back to text.")
-            raise ValueError(f"Unsupported MIME type: {mime_type}")
+
         
         try:
             with open(file_path, 'rb') as file:
@@ -263,7 +320,6 @@ class OpenAILLM(BaseLLM):
             
             return file_id
         except Exception as e:
-            import logging
             logging.error(f"Error uploading file to OpenAI: {e}")
             raise