Place conversation evidence before the current turn for prompt caching
The summary ledger is regenerated every turn. At the head of history it invalidated the cached retained turns on every request once trimming started. Insert it immediately before the current turn's HumanMessage so system, tools and retained history stay byte-identical; excerpts of dropped turns stay at the head since they only change when the cut moves. Reserve a fixed ledger ceiling (SUMMARY_RESERVE_TOKENS) rather than its current size, so a shrinking ledger cannot pull the cut back and change the retained prefix; fall back to the actual size for tight budgets. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -31,6 +31,8 @@ TRUNCATION_NOTE = "\n[truncated: earlier context]"
|
||||
IMAGE_PLACEHOLDER = "[image from an earlier message]"
|
||||
# On overflow, drop old turns down to about this share of the history budget.
|
||||
TRIM_LOW_WATERMARK = 0.65
|
||||
# Upper bound on the summary ledger message (summary.STATE_TOKEN_LIMIT + wrapper).
|
||||
SUMMARY_RESERVE_TOKENS = 2000
|
||||
|
||||
|
||||
def estimate_tokens(value) -> int:
|
||||
@@ -487,23 +489,29 @@ def trim_history(
|
||||
if summary_cost > token_budget:
|
||||
logging.info("Omitting optional conversation summary: insufficient budget")
|
||||
return trimmed
|
||||
try:
|
||||
trimmed, _ = _trim(messages, token_budget, reserve=summary_cost)
|
||||
except ValueError:
|
||||
# Reserve a fixed ceiling, not the ledger's current size, so a shrinking
|
||||
# ledger can't pull the cut back and change the retained prefix.
|
||||
for reserve in dict.fromkeys(
|
||||
(max(summary_cost, SUMMARY_RESERVE_TOKENS), summary_cost)
|
||||
):
|
||||
try:
|
||||
with_room, _ = _trim(messages, token_budget, reserve=reserve)
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
# Historical evidence must never evict a current request that fits
|
||||
# without it. Required current-turn metadata retains priority.
|
||||
logging.info(
|
||||
"Omitting optional conversation summary to preserve current request"
|
||||
)
|
||||
return trimmed
|
||||
if (
|
||||
trimmed
|
||||
and isinstance(trimmed[0], AIMessage)
|
||||
and isinstance(trimmed[0].content, str)
|
||||
and trimmed[0].content.startswith("Earlier conversation excerpts")
|
||||
):
|
||||
trimmed = trimmed[1:]
|
||||
return [summary_message, *trimmed]
|
||||
# The ledger changes every turn, so place it just before the current turn:
|
||||
# system, tools and retained history then stay byte-identical for the
|
||||
# prompt cache. Turns start at a HumanMessage, so the last one opens the
|
||||
# current turn and no tool exchange is split.
|
||||
start = max(i for i, m in enumerate(with_room) if isinstance(m, HumanMessage))
|
||||
return [*with_room[:start], summary_message, *with_room[start:]]
|
||||
|
||||
|
||||
def build_context(
|
||||
|
||||
Reference in New Issue
Block a user