Pass images directly to the agent as multimodal content

Image messages previously ran two sequential LLM generations: a full
non-streamed vision call to describe the image, then a second agent call
with the description pasted in as text. The agent's llama-server model is
already vision-capable, so attach the image to the user message as an
OpenAI-style image_url content part and let the single agent call see it.
Roughly halves image-turn latency and streams from the start.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

blueprints/conversation/__init__.py

@@ -10,7 +10,7 @@ from quart_jwt_extended import (
 )
 
 import blueprints.users.models
-from utils.image_process import analyze_user_image
+from utils.image_process import image_content_block
 from utils.image_upload import ImageValidationError, process_image
 from utils.s3_client import generate_presigned_url as s3_presigned_url
 from utils.s3_client import get_image as s3_get_image
@@ -64,7 +64,7 @@ async def _build_system_prompt_with_memories(user_id: str) -> str:
 def _build_messages_payload(
     conversation,
     query_text: str,
-    image_description: str | None = None,
+    image_block: dict | None = None,
     system_prompt: str | None = None,
 ) -> list:
     recent_messages = (
@@ -80,11 +80,13 @@ def _build_messages_payload(
             text = f"[User sent an image]\n{text}"
         messages_payload.append({"role": role, "content": text})
 
-    # Build the current user message with optional image description
-    if image_description:
-        content = f"[Image analysis: {image_description}]"
-        if query_text:
-            content = f"{query_text}\n\n{content}"
+    # Attach the image directly to the current message so the (multimodal)
+    # agent model sees it in one call, instead of a separate describe pass.
+    if image_block:
+        content = [
+            {"type": "text", "text": query_text or "The user sent this image."},
+            image_block,
+        ]
     else:
         content = query_text
     messages_payload.append({"role": "user", "content": content})
@@ -190,20 +192,19 @@ async def stream_query():
         image_key=image_key,
     )
 
-    # If an image was uploaded, analyze it with the vision model
-    image_description = None
+    # If an image was uploaded, attach it to the agent request directly
+    image_block = None
     if image_key:
         try:
             image_bytes, _ = await s3_get_image(image_key)
-            image_description = await analyze_user_image(image_bytes)
-            logging.info(f"Image analysis complete for {image_key}")
+            image_block = image_content_block(image_bytes)
         except Exception as e:
-            logging.error(f"Failed to analyze image: {e}")
-            image_description = "[Image could not be analyzed]"
+            logging.error(f"Failed to load image {image_key}: {e}")
+            query_text = f"{query_text or ''}\n\n[An image was sent but could not be loaded.]".strip()
 
     system_prompt = await _build_system_prompt_with_memories(str(user.id))
     messages_payload = _build_messages_payload(
-        conversation, query_text or "", image_description, system_prompt=system_prompt
+        conversation, query_text or "", image_block, system_prompt=system_prompt
     )
     payload = {"messages": messages_payload}
     agent_config = {

blueprints/imessage/__init__.py

@@ -200,21 +200,19 @@ async def webhook():
         user=user,
     )
 
-    # Analyze image if present
-    image_description = None
+    # Attach image directly to the agent request if present
+    image_block = None
     if media_url:
         try:
-            from utils.image_process import analyze_user_image
+            from utils.image_process import image_content_block
 
             async with httpx.AsyncClient() as client:
                 resp = await client.get(media_url, timeout=30)
                 resp.raise_for_status()
-                image_bytes = resp.content
-            image_description = await analyze_user_image(image_bytes)
-            logger.info(f"Image analysis complete for iMessage from {from_number}")
+            image_block = image_content_block(resp.content)
         except Exception as e:
-            logger.error(f"Failed to analyze iMessage image: {e}")
-            image_description = "[Image could not be analyzed]"
+            logger.error(f"Failed to load iMessage image: {e}")
+            content = f"{content or ''}\n\n[An image was sent but could not be loaded.]".strip()
 
     # Build messages payload for LangChain agent
     try:
@@ -227,10 +225,11 @@ async def webhook():
             role = "user" if msg.speaker == "user" else "assistant"
             messages_payload.append({"role": role, "content": msg.text})
 
-        if image_description:
-            user_content = f"[Image analysis: {image_description}]"
-            if content:
-                user_content = f"{content}\n\n{user_content}"
+        if image_block:
+            user_content = [
+                {"type": "text", "text": content or "The user sent this image."},
+                image_block,
+            ]
         else:
             user_content = content
         messages_payload.append({"role": "user", "content": user_content})

utils/image_process.py

@@ -87,50 +87,20 @@ def _to_jpeg_bytes(file_bytes: bytes) -> bytes:
     return buf.getvalue()
 
 
-async def analyze_user_image(file_bytes: bytes) -> str:
-    """Analyze an image uploaded by a user and return a text description.
+def image_content_block(file_bytes: bytes) -> dict:
+    """Build an OpenAI-style image_url content part from raw image bytes.
 
-    Uses llama-server (OpenAI-compatible API) with vision support.
-    Falls back to OpenAI if llama-server is not configured.
+    Normalizes to JPEG so HEIC/HEIF and other formats work, then encodes as a
+    base64 data URL suitable for a multimodal chat message.
     """
     import base64
 
-    from openai import AsyncOpenAI
-
-    file_bytes = _to_jpeg_bytes(file_bytes)
-
-    llama_url = os.getenv("LLAMA_SERVER_URL")
-    if llama_url:
-        aclient = AsyncOpenAI(base_url=llama_url, api_key="not-needed")
-        model = os.getenv("LLAMA_MODEL_NAME", "llama-3.1-8b-instruct")
-    else:
-        aclient = AsyncOpenAI()
-        model = "gpt-4o-mini"
-
-    b64 = base64.b64encode(file_bytes).decode("utf-8")
-
-    response = await aclient.chat.completions.create(
-        model=model,
-        messages=[
-            {
-                "role": "system",
-                "content": "You are a helpful image analyst. Describe what you see in the image in detail. Be thorough but concise.",
-            },
-            {
-                "role": "user",
-                "content": [
-                    {"type": "text", "text": "Please describe this image in detail."},
-                    {
-                        "type": "image_url",
-                        "image_url": {
-                            "url": f"data:image/jpeg;base64,{b64}",
-                        },
-                    },
-                ],
-            },
-        ],
-    )
-    return response.choices[0].message.content
+    jpeg_bytes = _to_jpeg_bytes(file_bytes)
+    b64 = base64.b64encode(jpeg_bytes).decode("utf-8")
+    return {
+        "type": "image_url",
+        "image_url": {"url": f"data:image/jpeg;base64,{b64}"},
+    }
 
 
 if __name__ == "__main__":