Add image upload and vision analysis to Ask Simba chat

Users can now attach images in the web chat for Simba to analyze using
Ollama's gemma3 vision model. Images are stored in Garage (S3-compatible)
and displayed in chat history.

Also fixes aerich migration config by extracting TORTOISE_CONFIG into a
standalone config/db.py module, removing the stale aerich_config.py, and
adding missing MODELS_STATE to migration 3.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-04 08:03:19 -04:00
parent ac9c821ec7
commit 0415610d64
17 changed files with 501 additions and 58 deletions

View File

@@ -14,6 +14,7 @@ import catIcon from "../assets/cat.png";
type Message = {
text: string;
speaker: "simba" | "user" | "tool";
image_key?: string | null;
};
type Conversation = {
@@ -55,6 +56,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
const [isLoading, setIsLoading] = useState<boolean>(false);
const [isAdmin, setIsAdmin] = useState<boolean>(false);
const [showAdminPanel, setShowAdminPanel] = useState<boolean>(false);
const [pendingImage, setPendingImage] = useState<File | null>(null);
const messagesEndRef = useRef<HTMLDivElement>(null);
const isMountedRef = useRef<boolean>(true);
@@ -80,7 +82,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
try {
const fetched = await conversationService.getConversation(conversation.id);
setMessages(
fetched.messages.map((m) => ({ text: m.text, speaker: m.speaker })),
fetched.messages.map((m) => ({ text: m.text, speaker: m.speaker, image_key: m.image_key })),
);
} catch (err) {
console.error("Failed to load messages:", err);
@@ -120,7 +122,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
try {
const conv = await conversationService.getConversation(selectedConversation.id);
setSelectedConversation({ id: conv.id, title: conv.name });
setMessages(conv.messages.map((m) => ({ text: m.text, speaker: m.speaker })));
setMessages(conv.messages.map((m) => ({ text: m.text, speaker: m.speaker, image_key: m.image_key })));
} catch (err) {
console.error("Failed to load messages:", err);
}
@@ -129,7 +131,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
}, [selectedConversation?.id]);
const handleQuestionSubmit = async () => {
if (!query.trim() || isLoading) return;
if ((!query.trim() && !pendingImage) || isLoading) return;
let activeConversation = selectedConversation;
if (!activeConversation) {
@@ -139,9 +141,13 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
setConversations((prev) => [activeConversation!, ...prev]);
}
// Capture pending image before clearing state
const imageFile = pendingImage;
const currMessages = messages.concat([{ text: query, speaker: "user" }]);
setMessages(currMessages);
setQuery("");
setPendingImage(null);
setIsLoading(true);
if (simbaMode) {
@@ -155,6 +161,29 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
abortControllerRef.current = abortController;
try {
// Upload image first if present
let imageKey: string | undefined;
if (imageFile) {
const uploadResult = await conversationService.uploadImage(
imageFile,
activeConversation.id,
);
imageKey = uploadResult.image_key;
// Update the user message with the image key
setMessages((prev) => {
const updated = [...prev];
// Find the last user message we just added
for (let i = updated.length - 1; i >= 0; i--) {
if (updated[i].speaker === "user") {
updated[i] = { ...updated[i], image_key: imageKey };
break;
}
}
return updated;
});
}
await conversationService.streamQuery(
query,
activeConversation.id,
@@ -170,6 +199,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
}
},
abortController.signal,
imageKey,
);
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
@@ -349,6 +379,9 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
handleQuestionSubmit={handleQuestionSubmit}
setSimbaMode={setSimbaMode}
isLoading={isLoading}
pendingImage={pendingImage}
onImageSelect={(file) => setPendingImage(file)}
onClearImage={() => setPendingImage(null)}
/>
</div>
</div>
@@ -375,7 +408,7 @@ export const ChatScreen = ({ setAuthenticated }: ChatScreenProps) => {
return <ToolBubble key={index} text={msg.text} />;
if (msg.speaker === "simba")
return <AnswerBubble key={index} text={msg.text} />;
return <QuestionBubble key={index} text={msg.text} />;
return <QuestionBubble key={index} text={msg.text} image_key={msg.image_key} />;
})}
{isLoading && <AnswerBubble text="" loading={true} />}