ryan
4ce254193e
The 30s time-to-first-token guard measured from the turn's first model call to the first answer token, so thinking, tool execution and the second call's prefill all counted against it. Long reasoning on a long conversation was cancelled mid-generation (llama-server: stop: cancel task at n_gen=1287). - stream_query: the timeout now fires only when an in-flight model call goes silent for TTFT_TIMEOUT_SECONDS. Any chunk (thinking or empty) resets the clock, each model call starts a fresh one, and time spent in tools between calls doesn't count. - Trim replayed history (blueprints/conversation/context.py): tool results and user messages from earlier turns are capped at 4000 chars (attachments included) and their images replaced by a placeholder. The current turn is untouched. The window no longer starts with orphaned ToolMessages. - Log per-call prefill time, duration and prompt/output tokens (stream_usage=True) plus max_prompt_tokens in the stream_request line. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
79 lines
2.5 KiB
Python
79 lines
2.5 KiB
Python
"""Tests for trimming the replayed history (blueprints/conversation/context.py)."""
|
|
|
|
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
|
|
|
from blueprints.conversation.context import (
|
|
EARLIER_MESSAGE_MAX_CHARS,
|
|
HISTORY_WINDOW,
|
|
IMAGE_PLACEHOLDER,
|
|
TRUNCATION_NOTE,
|
|
trim_history,
|
|
)
|
|
|
|
BIG = "x" * (EARLIER_MESSAGE_MAX_CHARS * 3)
|
|
|
|
|
|
def _call(call_id: str, name: str = "simba_search") -> AIMessage:
|
|
return AIMessage(content="", tool_calls=[{"id": call_id, "name": name, "args": {}}])
|
|
|
|
|
|
def _result(call_id: str, text: str = BIG) -> ToolMessage:
|
|
return ToolMessage(content=text, tool_call_id=call_id, name="simba_search")
|
|
|
|
|
|
def test_earlier_tool_results_are_capped_current_turn_untouched():
|
|
messages = [
|
|
HumanMessage(content="first question"),
|
|
_call("c1"),
|
|
_result("c1"),
|
|
AIMessage(content="first answer"),
|
|
HumanMessage(content="second question"),
|
|
_call("c2"),
|
|
_result("c2"),
|
|
]
|
|
|
|
out = trim_history(messages)
|
|
|
|
earlier, current = out[2], out[6]
|
|
assert earlier.content == BIG[:EARLIER_MESSAGE_MAX_CHARS] + TRUNCATION_NOTE
|
|
assert current.content == BIG
|
|
# The checkpointed originals are never mutated.
|
|
assert messages[2].content == BIG
|
|
|
|
|
|
def test_earlier_attachments_and_images_are_shrunk():
|
|
image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
|
|
messages = [
|
|
HumanMessage(content=[{"type": "text", "text": BIG}, image]),
|
|
AIMessage(content="nice photo"),
|
|
HumanMessage(content=[{"type": "text", "text": "now this one"}, image]),
|
|
]
|
|
|
|
out = trim_history(messages)
|
|
|
|
earlier = out[0].content
|
|
assert earlier[0]["text"].endswith(TRUNCATION_NOTE)
|
|
assert earlier[1] == {"type": "text", "text": IMAGE_PLACEHOLDER}
|
|
# The current turn keeps its image.
|
|
assert out[2].content[1] == image
|
|
|
|
|
|
def test_window_never_starts_with_orphaned_tool_results():
|
|
# Many parallel tool results so the window cut lands between an AIMessage's
|
|
# tool_calls and their ToolMessages.
|
|
messages = [HumanMessage(content="q"), _call("c0")]
|
|
messages += [_result("c0", "r") for _ in range(HISTORY_WINDOW)]
|
|
messages += [AIMessage(content="answer"), HumanMessage(content="next")]
|
|
|
|
out = trim_history(messages)
|
|
|
|
assert len(out) <= HISTORY_WINDOW
|
|
assert not isinstance(out[0], ToolMessage)
|
|
|
|
|
|
def test_short_history_is_returned_as_is():
|
|
messages = [HumanMessage(content="hi"), AIMessage(content="hello")]
|
|
out = trim_history(messages)
|
|
assert out == messages
|
|
assert out[0] is messages[0]
|