Budget conversation context while preserving current turns and tool exchanges

2026-10-12 19:45:51 -04:00
parent 2cb131bbd2
commit 1c9405cfdf
7 changed files with 632 additions and 107 deletions
@@ -27,6 +27,11 @@ LLAMA_MODEL_NAME=llama-3.1-8b-instruct
 # Abandon a turn when an in-flight model call produces no output (prefill included)
 # for this long. Thinking tokens count as output, so long reasoning is fine.
 TTFT_TIMEOUT_SECONDS=30
+# Context estimates include system instructions and tool definitions.
+# Set the window to the smallest backend context; leave room for generation.
+CONTEXT_WINDOW_TOKENS=32768
+PROMPT_BUDGET_TOKENS=16000
+RESPONSE_RESERVE_TOKENS=4096

 # Embedding Server Configuration
 # If set, uses a custom OpenAI-compatible embedding server (e.g. llama-server)