Add image upload and vision analysis to Ask Simba chat

Users can now attach images in the web chat for Simba to analyze using Ollama's gemma3 vision model. Images are stored in Garage (S3-compatible) and displayed in chat history. Also fixes aerich migration config by extracting TORTOISE_CONFIG into a standalone config/db.py module, removing the stale aerich_config.py, and adding missing MODELS_STATE to migration 3. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-04 08:03:19 -04:00
parent ac9c821ec7
commit 0415610d64
17 changed files with 501 additions and 58 deletions
--- a/raggr-frontend/src/components/ChatScreen.tsx
+++ b/raggr-frontend/src/components/ChatScreen.tsx
@@ -14,6 +14,7 @@ import catIcon from "../assets/cat.png";
 type Message = {
  text: string;
  speaker: "simba" | "user" | "tool";
+  image_key?: string | null;
 };

 type Conversation = {
@@ -55,6 +56,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
  const [isLoading, setIsLoading] = useState<boolean>(false);
  const [isAdmin, setIsAdmin] = useState<boolean>(false);
  const [showAdminPanel, setShowAdminPanel] = useState<boolean>(false);
+  const [pendingImage, setPendingImage] = useState<File | null>(null);

  const messagesEndRef = useRef<HTMLDivElement>(null);
  const isMountedRef = useRef<boolean>(true);
@@ -80,7 +82,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
      try {
        const fetched = await conversationService.getConversation(conversation.id);
        setMessages(
-          fetched.messages.map((m) => ({ text: m.text, speaker: m.speaker })),
+          fetched.messages.map((m) => ({ text: m.text, speaker: m.speaker, image_key: m.image_key })),
        );
      } catch (err) {
        console.error("Failed to load messages:", err);
@@ -120,7 +122,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
      try {
        const conv = await conversationService.getConversation(selectedConversation.id);
        setSelectedConversation({ id: conv.id, title: conv.name });
-        setMessages(conv.messages.map((m) => ({ text: m.text, speaker: m.speaker })));
+        setMessages(conv.messages.map((m) => ({ text: m.text, speaker: m.speaker, image_key: m.image_key })));
      } catch (err) {
        console.error("Failed to load messages:", err);
      }
@@ -129,7 +131,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
  }, [selectedConversation?.id]);

  const handleQuestionSubmit = async () => {
-    if (!query.trim() || isLoading) return;
+    if ((!query.trim() && !pendingImage) || isLoading) return;

    let activeConversation = selectedConversation;
    if (!activeConversation) {
@@ -139,9 +141,13 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
      setConversations((prev) => [activeConversation!, ...prev]);
    }

+    // Capture pending image before clearing state
+    const imageFile = pendingImage;
+
    const currMessages = messages.concat([{ text: query, speaker: "user" }]);
    setMessages(currMessages);
    setQuery("");
+    setPendingImage(null);
    setIsLoading(true);

    if (simbaMode) {
@@ -155,6 +161,29 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
    abortControllerRef.current = abortController;

    try {
+      // Upload image first if present
+      let imageKey: string | undefined;
+      if (imageFile) {
+        const uploadResult = await conversationService.uploadImage(
+          imageFile,
+          activeConversation.id,
+        );
+        imageKey = uploadResult.image_key;
+
+        // Update the user message with the image key
+        setMessages((prev) => {
+          const updated = [...prev];
+          // Find the last user message we just added
+          for (let i = updated.length - 1; i >= 0; i--) {
+            if (updated[i].speaker === "user") {
+              updated[i] = { ...updated[i], image_key: imageKey };
+              break;
+            }
+          }
+          return updated;
+        });
+      }
+
      await conversationService.streamQuery(
        query,
        activeConversation.id,
@@ -170,6 +199,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
          }
        },
        abortController.signal,
+        imageKey,
      );
    } catch (error) {
      if (error instanceof Error && error.name === "AbortError") {
@@ -349,6 +379,9 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
                handleQuestionSubmit={handleQuestionSubmit}
                setSimbaMode={setSimbaMode}
                isLoading={isLoading}
+                pendingImage={pendingImage}
+                onImageSelect={(file) => setPendingImage(file)}
+                onClearImage={() => setPendingImage(null)}
              />
            </div>
          </div>
@@ -375,7 +408,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
                    return <ToolBubble key={index} text={msg.text} />;
                  if (msg.speaker === "simba")
                    return <AnswerBubble key={index} text={msg.text} />;
-                  return <QuestionBubble key={index} text={msg.text} />;
+                  return <QuestionBubble key={index} text={msg.text} image_key={msg.image_key} />;
                })}

                {isLoading && <AnswerBubble text="" loading={true} />}
--- a/raggr-frontend/src/components/MessageInput.tsx
+++ b/raggr-frontend/src/components/MessageInput.tsx
@@ -1,5 +1,5 @@
-import { useState } from "react";
-import { ArrowUp } from "lucide-react";
+import { useRef, useState } from "react";
+import { ArrowUp, ImagePlus, X } from "lucide-react";
 import { cn } from "../lib/utils";
 import { Textarea } from "./ui/textarea";

@@ -10,6 +10,9 @@ type MessageInputProps = {
  setSimbaMode: (val: boolean) => void;
  query: string;
  isLoading: boolean;
+  pendingImage: File | null;
+  onImageSelect: (file: File) => void;
+  onClearImage: () => void;
 };

 export const MessageInput = ({
@@ -19,8 +22,12 @@ export const MessageInput = ({
  handleQuestionSubmit,
  setSimbaMode,
  isLoading,
+  pendingImage,
+  onImageSelect,
+  onClearImage,
 }: MessageInputProps) => {
  const [simbaMode, setLocalSimbaMode] = useState(false);
+  const fileInputRef = useRef<HTMLInputElement>(null);

  const toggleSimbaMode = () => {
    const next = !simbaMode;
@@ -28,6 +35,17 @@ export const MessageInput = ({
    setSimbaMode(next);
  };

+  const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+    if (file) {
+      onImageSelect(file);
+    }
+    // Reset so the same file can be re-selected
+    e.target.value = "";
+  };
+
+  const canSend = !isLoading && (query.trim() || pendingImage);
+
  return (
    <div
      className={cn(
@@ -36,6 +54,26 @@ export const MessageInput = ({
        "focus-within:border-amber-soft/60",
      )}
    >
+      {/* Image preview */}
+      {pendingImage && (
+        <div className="px-3 pt-3">
+          <div className="relative inline-block">
+            <img
+              src={URL.createObjectURL(pendingImage)}
+              alt="Pending upload"
+              className="h-20 rounded-lg object-cover border border-sand"
+            />
+            <button
+              type="button"
+              onClick={onClearImage}
+              className="absolute -top-1.5 -right-1.5 w-5 h-5 rounded-full bg-charcoal text-white flex items-center justify-center hover:bg-charcoal/80 transition-colors cursor-pointer"
+            >
+              <X size={12} />
+            </button>
+          </div>
+        </div>
+      )}
+
      {/* Textarea */}
      <Textarea
        onChange={handleQueryChange}
@@ -46,32 +84,58 @@ export const MessageInput = ({
        className="min-h-[60px] max-h-40"
      />

+      {/* Hidden file input */}
+      <input
+        ref={fileInputRef}
+        type="file"
+        accept="image/*"
+        onChange={handleFileChange}
+        className="hidden"
+      />
+
      {/* Bottom toolbar */}
      <div className="flex items-center justify-between px-3 pb-2.5 pt-1">
-        {/* Simba mode toggle */}
-        <button
-          type="button"
-          onClick={toggleSimbaMode}
-          className="flex items-center gap-2 group cursor-pointer select-none"
-        >
-          <div className={cn("toggle-track", simbaMode && "checked")}>
-            <div className="toggle-thumb" />
-          </div>
-          <span className="text-xs text-warm-gray group-hover:text-charcoal transition-colors">
-            simba mode
-          </span>
-        </button>
+        <div className="flex items-center gap-3">
+          {/* Simba mode toggle */}
+          <button
+            type="button"
+            onClick={toggleSimbaMode}
+            className="flex items-center gap-2 group cursor-pointer select-none"
+          >
+            <div className={cn("toggle-track", simbaMode && "checked")}>
+              <div className="toggle-thumb" />
+            </div>
+            <span className="text-xs text-warm-gray group-hover:text-charcoal transition-colors">
+              simba mode
+            </span>
+          </button>
+
+          {/* Image attach button */}
+          <button
+            type="button"
+            onClick={() => fileInputRef.current?.click()}
+            disabled={isLoading}
+            className={cn(
+              "w-7 h-7 rounded-lg flex items-center justify-center transition-all cursor-pointer",
+              isLoading
+                ? "text-warm-gray/40 cursor-not-allowed"
+                : "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
+            )}
+          >
+            <ImagePlus size={16} />
+          </button>
+        </div>

        {/* Send button */}
        <button
          type="submit"
          onClick={handleQuestionSubmit}
-          disabled={isLoading || !query.trim()}
+          disabled={!canSend}
          className={cn(
            "w-8 h-8 rounded-full flex items-center justify-center",
            "transition-all duration-200 cursor-pointer",
            "shadow-sm",
-            isLoading || !query.trim()
+            !canSend
              ? "bg-sand text-warm-gray/50 cursor-not-allowed shadow-none"
              : "bg-amber-glow text-white hover:bg-amber-dark hover:shadow-md hover:shadow-amber-glow/30 active:scale-95",
          )}
--- a/raggr-frontend/src/components/QuestionBubble.tsx
+++ b/raggr-frontend/src/components/QuestionBubble.tsx
@@ -1,10 +1,12 @@
 import { cn } from "../lib/utils";
+import { conversationService } from "../api/conversationService";

 type QuestionBubbleProps = {
  text: string;
+  image_key?: string | null;
 };

-export const QuestionBubble = ({ text }: QuestionBubbleProps) => {
+export const QuestionBubble = ({ text, image_key }: QuestionBubbleProps) => {
  return (
    <div className="flex justify-end message-enter">
      <div
@@ -15,6 +17,13 @@ export const QuestionBubble = ({ text }: QuestionBubbleProps) => {
          "shadow-sm shadow-leaf/10",
        )}
      >
+        {image_key && (
+          <img
+            src={conversationService.getImageUrl(image_key)}
+            alt="Uploaded image"
+            className="max-w-full rounded-xl mb-2"
+          />
+        )}
        {text}
      </div>
    </div>