From b361377088a4dcb8c81a2ddb2cc9870e3faa4067 Mon Sep 17 00:00:00 2001 From: Ryan Chen Date: Fri, 11 Sep 2026 17:04:45 -0400 Subject: [PATCH] Add read timeouts to llama-server clients so hangs fail fast The OpenAI client defaults to a 600s read timeout, so a wedged llama-server blocked requests for 10 minutes and never raised, which meant with_fallbacks() to OpenAI never triggered. Set a per-chunk read timeout (120s, configurable via LLAMA_TIMEOUT_SECONDS) plus one retry on the ChatOpenAI and LLMClient clients. Context is preserved because conversation history is rebuilt from Postgres on every request. --- blueprints/conversation/agents.py | 10 ++++++++++ llm.py | 11 ++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/blueprints/conversation/agents.py b/blueprints/conversation/agents.py index 0291f2a..b8b062e 100644 --- a/blueprints/conversation/agents.py +++ b/blueprints/conversation/agents.py @@ -1,6 +1,7 @@ import os from typing import cast +import httpx from dotenv import load_dotenv from langchain.agents import create_agent from langchain.chat_models import BaseChatModel @@ -20,12 +21,21 @@ from utils.ynab_service import YNABService load_dotenv() # Configure LLM with llama-server or OpenAI fallback +# Read timeout between streamed chunks. The OpenAI client defaults to 600s, +# which means a wedged llama-server blocks a request for 10 minutes before +# erroring. With a tight timeout, a hang raises quickly so +# with_fallbacks() below kicks in and the conversation (rebuilt from the +# DB each request) continues seamlessly on OpenAI. +LLAMA_READ_TIMEOUT = float(os.getenv("LLAMA_TIMEOUT_SECONDS", "120")) + llama_url = os.getenv("LLAMA_SERVER_URL") if llama_url: llama_chat = ChatOpenAI( base_url=llama_url, api_key="not-needed", model=os.getenv("LLAMA_MODEL_NAME", "llama-3.1-8b-instruct"), + timeout=httpx.Timeout(LLAMA_READ_TIMEOUT, connect=10.0), + max_retries=1, ) else: llama_chat = None diff --git a/llm.py b/llm.py index 1417a6a..cfac6c1 100644 --- a/llm.py +++ b/llm.py @@ -1,6 +1,7 @@ import os import logging +import httpx from openai import OpenAI from dotenv import load_dotenv @@ -13,7 +14,15 @@ class LLMClient: def __init__(self): llama_url = os.getenv("LLAMA_SERVER_URL") if llama_url: - self.client = OpenAI(base_url=llama_url, api_key="not-needed") + # Tight read timeout so a wedged llama-server fails fast instead + # of hanging for the OpenAI client's 600s default. + read_timeout = float(os.getenv("LLAMA_TIMEOUT_SECONDS", "120")) + self.client = OpenAI( + base_url=llama_url, + api_key="not-needed", + timeout=httpx.Timeout(read_timeout, connect=10.0), + max_retries=1, + ) self.model = os.getenv("LLAMA_MODEL_NAME", "llama-3.1-8b-instruct") self.PROVIDER = "llama_server" logging.info("Using llama_server as LLM backend")