Merge pull request 'Add read timeouts to llama-server clients so hangs fail fast' (#43) from llama-timeouts into main

Reviewed-on: #43

CLAUDE.md

@@ -124,6 +124,16 @@ make test       # Run tests
 make test-cov   # Run tests with coverage
 ```
 
+Note: `make test` may fail locally (no pytest on PATH); use `.venv/bin/pytest tests/ -q` instead.
+
+## Git Workflow
+
+- The host runs **Gitea** (`git.torrtle.co`), not GitHub, despite the remote name. Use the **`tea` CLI** for PRs (Gitea flavor: `tea pr create`, `tea pr list`, etc. — no `tea create`).
+- Typical flow: commit locally, `git push -u origin <branch>`, then `tea pr create -t "title" -d "body"`.
+- "kermit" is the user's slang for **commit** (e.g. "kermit and push" = commit and push). There is no kermit tool.
+- Keep one logical change per branch/PR; don't ride along on stale feature branches.
+- Don't commit `.DS_Store`, `chroma_db/`, `chromadb/`, `.planning/` (local artifacts).
+
 ## Key Patterns
 
 - All endpoints are async (`async def`)

blueprints/conversation/agents.py

@@ -1,6 +1,7 @@
 import os
 from typing import cast
 
+import httpx
 from dotenv import load_dotenv
 from langchain.agents import create_agent
 from langchain.chat_models import BaseChatModel
@@ -20,12 +21,21 @@ from utils.ynab_service import YNABService
 load_dotenv()
 
 # Configure LLM with llama-server or OpenAI fallback
+# Read timeout between streamed chunks. The OpenAI client defaults to 600s,
+# which means a wedged llama-server blocks a request for 10 minutes before
+# erroring. With a tight timeout, a hang raises quickly so
+# with_fallbacks() below kicks in and the conversation (rebuilt from the
+# DB each request) continues seamlessly on OpenAI.
+LLAMA_READ_TIMEOUT = float(os.getenv("LLAMA_TIMEOUT_SECONDS", "120"))
+
 llama_url = os.getenv("LLAMA_SERVER_URL")
 if llama_url:
     llama_chat = ChatOpenAI(
         base_url=llama_url,
         api_key="not-needed",
         model=os.getenv("LLAMA_MODEL_NAME", "llama-3.1-8b-instruct"),
+        timeout=httpx.Timeout(LLAMA_READ_TIMEOUT, connect=10.0),
+        max_retries=1,
     )
 else:
     llama_chat = None

llm.py

@@ -1,6 +1,7 @@
 import os
 import logging
 
+import httpx
 from openai import OpenAI
 from dotenv import load_dotenv
 
@@ -13,7 +14,15 @@ class LLMClient:
     def __init__(self):
         llama_url = os.getenv("LLAMA_SERVER_URL")
         if llama_url:
-            self.client = OpenAI(base_url=llama_url, api_key="not-needed")
+            # Tight read timeout so a wedged llama-server fails fast instead
+            # of hanging for the OpenAI client's 600s default.
+            read_timeout = float(os.getenv("LLAMA_TIMEOUT_SECONDS", "120"))
+            self.client = OpenAI(
+                base_url=llama_url,
+                api_key="not-needed",
+                timeout=httpx.Timeout(read_timeout, connect=10.0),
+                max_retries=1,
+            )
             self.model = os.getenv("LLAMA_MODEL_NAME", "llama-3.1-8b-instruct")
             self.PROVIDER = "llama_server"
             logging.info("Using llama_server as LLM backend")