ryan
891f44c1a9
llama-server only reuses a byte-identical prompt prefix. Several things changed the prefix every turn, forcing most of the prompt to be re-prefilled: - Memories: rank still chooses which memories fit the budget, but the chosen ones are emitted in stored order, so the block is identical across queries. - History trimming: the conversation-evidence summary is only included when turns were actually dropped (it duplicated retained turns otherwise). On overflow, old turns are dropped at stable cut points down to ~65% of the budget, so the retained prefix holds for several turns instead of shifting by one turn every request. - Tools: TOOL_ROUTING_MODE=static (default) binds the full role-appropriate tool set in registration order with no discovery tool; admin gating is unchanged. TOOL_ROUTING_MODE=dynamic keeps the keyword router. - Observability: per-call model_calls in the stream_request log now include cached_prompt_tokens when the provider reports cache reads. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
81 lines
3.2 KiB
Python
81 lines
3.2 KiB
Python
from collections import Counter
|
|
|
|
from .models import UserMemory
|
|
from .prompts import SIMBA_SYSTEM_PROMPT
|
|
from .context import estimate_tokens
|
|
from .tool_routing import words
|
|
|
|
MEMORY_BUDGET_TOKENS = 1000
|
|
|
|
|
|
async def get_memories_for_user(user_id: str) -> list[str]:
|
|
"""Return all memory content strings for a user, ordered by most recently updated."""
|
|
memories = await UserMemory.filter(user_id=user_id).order_by("-updated_at")
|
|
return [m.content for m in memories]
|
|
|
|
|
|
def rank_memories(memories: list[str], query: str = "") -> list[str]:
|
|
"""Rank lexical relevance first, with a bounded recency bonus.
|
|
|
|
Input order is newest first. No database fields or stored memories change.
|
|
"""
|
|
stop_words = words(
|
|
"a an the and or of for to in on is are was were i me my you your what how do does can please about remember"
|
|
)
|
|
terms = words(query) - stop_words
|
|
if not terms:
|
|
return list(memories)
|
|
|
|
# Normalize simple plurals to match e.g. recipes/recipe without dependencies.
|
|
def normalize(tokens):
|
|
return {t[:-1] if len(t) > 3 and t.endswith("s") else t for t in tokens}
|
|
|
|
terms = normalize(terms)
|
|
scored = []
|
|
for index, memory in enumerate(memories):
|
|
overlap = len(terms & normalize(words(memory))) / len(terms)
|
|
score = (1 + overlap if overlap else 0) + 0.1 / (index + 1)
|
|
scored.append((score, index, memory))
|
|
return [m for _, _, m in sorted(scored, key=lambda row: (-row[0], row[1]))]
|
|
|
|
|
|
async def build_system_prompt_with_memories(user_id: str, query: str = "") -> str:
|
|
memories = await get_memories_for_user(user_id)
|
|
if not memories:
|
|
return SIMBA_SYSTEM_PROMPT
|
|
header = "\n\nUSER MEMORIES (facts the user has asked you to remember):\n"
|
|
omission = "\n[Additional saved memories omitted from this prompt.]"
|
|
# Reserve wrapper and omission costs so the entire injected memory block fits.
|
|
used = estimate_tokens(header) + estimate_tokens(omission)
|
|
chosen = Counter()
|
|
for memory in rank_memories(memories, query):
|
|
cost = estimate_tokens(f"- {memory}\n")
|
|
if used + cost <= MEMORY_BUDGET_TOKENS:
|
|
chosen[memory] += 1
|
|
used += cost
|
|
# Rank only chooses what fits; emit in stored order so the system prompt is
|
|
# byte-identical across queries and llama-server can reuse its KV cache.
|
|
selected = []
|
|
for memory in memories:
|
|
if chosen[memory]:
|
|
chosen[memory] -= 1
|
|
selected.append(memory)
|
|
block = "\n".join(f"- {m}" for m in selected)
|
|
if len(selected) < len(memories):
|
|
block += omission
|
|
if not selected:
|
|
return SIMBA_SYSTEM_PROMPT
|
|
return f"{SIMBA_SYSTEM_PROMPT}{header}{block}"
|
|
|
|
|
|
async def save_memory(user_id: str, content: str) -> str:
|
|
"""Save a new memory or touch an existing one (exact-match dedup)."""
|
|
existing = await UserMemory.filter(user_id=user_id, content=content).first()
|
|
if existing:
|
|
existing.updated_at = None # auto_now=True will refresh it on save
|
|
await existing.save(update_fields=["updated_at"])
|
|
return "Memory already exists (refreshed)."
|
|
|
|
await UserMemory.create(user_id=user_id, content=content)
|
|
return "Memory saved."
|