Add TTFT watchdog to detect and surface LLM stalls

When llama-server is queue-backed it accepts the connection but never
produces tokens; after ~60s Caddy drops the connection and the frontend
clears loading state with no response and no error shown to the user.

Backend: replace the async-for event loop with an explicit __anext__()
iterator wrapped in asyncio.wait_for(). A 30s timeout (TTFT_TIMEOUT_SECONDS)
applies only in the window between on_chat_model_start and the first content
token — tool calls are excluded. On timeout, emits a stalled error event and
logs it rather than waiting for the proxy to kill the connection.

Frontend: separates stall errors from generic errors; shows a "Simba didn't
respond in time. Retry" pill that re-sends the original query.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.env.example

@@ -18,6 +18,7 @@ BASE_URL=192.168.1.5:8000
 # If set, uses llama-server as the primary LLM backend with OpenAI as fallback
 LLAMA_SERVER_URL=http://192.168.1.213:8080/v1
 LLAMA_MODEL_NAME=llama-3.1-8b-instruct
+TTFT_TIMEOUT_SECONDS=30
 
 # Embedding Server Configuration
 # If set, uses a custom OpenAI-compatible embedding server (e.g. llama-server)

blueprints/conversation/__init__.py

@@ -1,6 +1,7 @@
 import asyncio
 import json
 import logging
+import os
 import time
 import uuid
 
@@ -32,6 +33,8 @@ from .models import (
 )
 from .prompts import SIMBA_SYSTEM_PROMPT
 
+TTFT_TIMEOUT = float(os.getenv("TTFT_TIMEOUT_SECONDS", "30"))
+
 conversation_blueprint = Blueprint(
     "conversation_api", __name__, url_prefix="/api/conversation"
 )
@@ -272,9 +275,34 @@ async def stream_query():
             )
 
         try:
-            async for event in main_agent.astream_events(
+            aiter = main_agent.astream_events(
                 payload, version="v2", config=agent_config
-            ):
+            ).__aiter__()
+            while True:
+                # Apply a short TTFT timeout only during the window between
+                # on_chat_model_start and the first content token. Before the
+                # LLM call starts (tool phase) and after the first token, let
+                # the httpx read timeout handle stalls instead.
+                if llm_start_at is not None and first_content_at is None:
+                    ttft_wait = TTFT_TIMEOUT - (time.monotonic() - llm_start_at)
+                    timeout = max(ttft_wait, 1.0)
+                else:
+                    timeout = None
+
+                try:
+                    if timeout is not None:
+                        event = await asyncio.wait_for(
+                            aiter.__anext__(), timeout=timeout
+                        )
+                    else:
+                        event = await aiter.__anext__()
+                except StopAsyncIteration:
+                    break
+                except asyncio.TimeoutError:
+                    _log_summary(error=f"TTFT timeout after {TTFT_TIMEOUT:.0f}s")
+                    yield f"data: {json.dumps({'type': 'error', 'message': 'Simba took too long to respond. Please try again.', 'stalled': True})}\n\n"
+                    return
+
                 event_type = event.get("event")
 
                 if event_type == "on_tool_start":

raggr-frontend/src/api/conversationService.ts

@@ -6,7 +6,7 @@ export type SSEEvent =
   | { type: "tool_end"; tool: string }
   | { type: "content"; delta: string }
   | { type: "response"; message: string }
-  | { type: "error"; message: string };
+  | { type: "error"; message: string; stalled?: boolean };
 
 export type SSEEventCallback = (event: SSEEvent) => void;
 

raggr-frontend/src/components/ChatScreen.tsx

@@ -61,6 +61,8 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => {
     stopGeneration,
     error,
     clearError,
+    stalled,
+    retryStalled,
   } = useChat({
     createConversation,
     refreshConversations,
@@ -273,6 +275,20 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => {
                   </div>
                 )}
 
+                {stalled && (
+                  <div className="flex justify-center message-enter">
+                    <div className="inline-flex items-center gap-2 px-3 py-1.5 rounded-full bg-red-50 border border-red-200 text-xs text-red-600">
+                      <span>Simba didn&apos;t respond in time.</span>
+                      <button
+                        onClick={retryStalled}
+                        className="font-medium underline underline-offset-2 hover:text-red-800 transition-colors cursor-pointer"
+                      >
+                        Retry
+                      </button>
+                    </div>
+                  </div>
+                )}
+
                 <div ref={messagesEndRef} />
               </div>
             </div>

raggr-frontend/src/hooks/useChat.ts

@@ -51,9 +51,12 @@ export function useChat({
   const [messagesLoading, setMessagesLoading] = useState(false);
   const [pendingImage, setPendingImage] = useState<File | null>(null);
   const [error, setError] = useState<string | null>(null);
+  const [stalled, setStalled] = useState(false);
   const [streaming, setStreaming] = useState(false);
   const [thinking, setThinking] = useState(false);
 
+  const stalledQueryRef = useRef<string | null>(null);
+
   const isMountedRef = useRef(true);
   // True once tokens for the current assistant reply have started arriving, so
   // new content appends to the live bubble instead of creating another one.
@@ -144,6 +147,8 @@ export function useChat({
       const imageFile = pendingImage;
 
       setError(null);
+      setStalled(false);
+      stalledQueryRef.current = null;
       streamingActiveRef.current = false;
       setStreaming(false);
       updateMessages((prev) => prev.concat([{ text: query, speaker: "user" }]));
@@ -239,7 +244,11 @@ export function useChat({
               }
             } else if (event.type === "error") {
               console.error("Stream error:", event.message);
-              if (isMountedRef.current) {
+              if (!isMountedRef.current) return;
+              if (event.stalled) {
+                stalledQueryRef.current = query;
+                setStalled(true);
+              } else {
                 setError("Simba ran into a problem answering that.");
               }
             }
@@ -293,6 +302,14 @@ export function useChat({
 
   const clearError = useCallback(() => setError(null), []);
 
+  const retryStalled = useCallback(() => {
+    const q = stalledQueryRef.current;
+    if (!q) return;
+    setStalled(false);
+    stalledQueryRef.current = null;
+    sendMessage(q, false);
+  }, [sendMessage]);
+
   return {
     messages,
     setMessages: updateMessages,
@@ -306,5 +323,7 @@ export function useChat({
     stopGeneration,
     error,
     clearError,
+    stalled,
+    retryStalled,
   };
 }