Files
simbarag/blueprints/conversation/__init__.py
T
ryan c329d15f7f
CI / backend (pull_request) Successful in 42s
CI / frontend (pull_request) Failing after 7s
CI / backend (push) Successful in 42s
CI / frontend (push) Failing after 7s
Prune LangGraph checkpoints per thread and delete them with the conversation
Every graph step stores a full snapshot of the conversation, nothing pruned
old ones, and deleting a conversation left its thread behind (about 1.6 MB per
checkpoint at 200 turns). After each turn, only the newest
CHECKPOINT_KEEP_PER_THREAD (default 20) checkpoints of that thread are kept,
plus their pending writes; state reads only use the newest, and pruning a
200-checkpoint thread takes about 2 ms. Deleting a conversation also deletes
its thread, without failing the request if the checkpoint store errors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 03:41:47 -04:00

770 lines
30 KiB
Python

import asyncio
import json
import logging
import os
import re
import time
import uuid
from urllib.parse import quote, unquote
from quart import Blueprint, current_app, jsonify, request
from quart_jwt_extended import get_jwt_identity
import blueprints.users.models
from blueprints.artifacts.logic import artifact_ref, create_artifact_from_output
from blueprints.artifacts.models import Artifact
from blueprints.users.decorators import jwt_access_required
from metrics import count_result, observe_response, time_response
from utils.image_process import image_content_block
from utils.strip_markdown import extract_agent_text
from utils.file_extract import (
FILE_TYPES,
FileExtractionError,
attachment_block,
extract_text,
file_extension,
)
from utils.image_upload import ALLOWED_TYPES as IMAGE_TYPES
from utils.image_upload import ImageValidationError, process_image
from utils.s3_client import generate_presigned_url as s3_presigned_url
from utils.s3_client import get_file as s3_get_file
from utils.s3_client import get_image as s3_get_image
from utils.s3_client import upload_file as s3_upload_file
from utils.s3_client import upload_image as s3_upload_image
from .checkpoints import delete_thread_checkpoints
from .logic import (
add_message_to_conversation,
persist_agent_tool_messages,
tool_result_text,
)
from . import runtime
from .runtime import prepare_summary as _prepare_conversation_summary # noqa: F401 — compatibility export
from .models import (
Conversation,
PydConversation,
PydListConversation,
Speaker,
)
from .tool_events import tool_end_payload, tool_name_from_message, tool_start_payload
from .tool_labels import tool_label
# Longest an in-flight model call may go without producing any output (prompt
# prefill included) before the turn is abandoned as stalled. Thinking tokens and
# empty chunks count as output; time spent running tools does not count at all.
MODEL_STALL_TIMEOUT = float(os.getenv("TTFT_TIMEOUT_SECONDS", "30"))
def stall_timeout(last_model_activity: float | None, now: float) -> float | None:
"""Seconds left before the in-flight model call counts as stalled.
None when no model call is in flight (e.g. while tools run): the httpx read
timeout and the tools' own timeouts cover those phases.
"""
if last_model_activity is None:
return None
return max(MODEL_STALL_TIMEOUT - (now - last_model_activity), 1.0)
def _model_call_summary(call: dict) -> dict:
"""Per-LLM-call timing and token counts for the stream_request log line."""
start = call["start"]
first_chunk = call.get("first_chunk")
end = call.get("end")
return {
"prefill_ms": round((first_chunk - start) * 1000) if first_chunk else None,
"ms": round((end - start) * 1000) if end else None,
"prompt_tokens": call.get("prompt_tokens"),
"cached_prompt_tokens": call.get("cached_prompt_tokens"),
"output_tokens": call.get("output_tokens"),
"model": call.get("model"),
"provider": call.get("provider"),
"reasoning_tokens": call.get("reasoning_tokens"),
}
_THINK_OPEN = "<think>"
_THINK_CLOSE = "</think>"
_THINK_RE = re.compile(r"<think>.*?</think>", re.DOTALL)
def _route_content(
buf: str, text: str, in_thinking: bool
) -> tuple[str, str, bool, str]:
"""Split a streaming content chunk into (thinking_text, content_text).
Handles tags split across chunk boundaries by keeping a small trailing
buffer. Returns the updated in_thinking state and any leftover buffer.
"""
combined = buf + text
thinking_parts: list[str] = []
content_parts: list[str] = []
while combined:
tag, hold = (
(_THINK_CLOSE, len(_THINK_CLOSE) - 1)
if in_thinking
else (_THINK_OPEN, len(_THINK_OPEN) - 1)
)
idx = combined.find(tag)
if idx == -1:
# Tag not yet complete — hold back enough bytes to catch a split.
if len(combined) <= hold:
return (
"".join(thinking_parts),
"".join(content_parts),
in_thinking,
combined,
)
safe, leftover = combined[:-hold], combined[-hold:]
(thinking_parts if in_thinking else content_parts).append(safe)
return (
"".join(thinking_parts),
"".join(content_parts),
in_thinking,
leftover,
)
(thinking_parts if in_thinking else content_parts).append(combined[:idx])
combined = combined[idx + len(tag) :]
in_thinking = not in_thinking
return "".join(thinking_parts), "".join(content_parts), in_thinking, ""
conversation_blueprint = Blueprint(
"conversation_api", __name__, url_prefix="/api/conversation"
)
async def _get_owned_conversation(conversation_id, user_id) -> Conversation | None:
"""Fetch a conversation only if it belongs to the given user."""
return await Conversation.get_or_none(id=conversation_id, user_id=user_id)
def _build_user_message(
query_text: str,
image_block: dict | None = None,
) -> dict:
"""Build the single new user message for this turn.
History is restored by the LangGraph checkpointer; only the current
message is passed to avoid duplicating prior turns in checkpoint state.
"""
if image_block:
content = [
{"type": "text", "text": query_text or "The user sent this image."},
image_block,
]
else:
content = query_text
return {"role": "user", "content": content}
@conversation_blueprint.post("/query")
@jwt_access_required
async def query():
current_user_uuid = get_jwt_identity()
user = await blueprints.users.models.User.get(id=current_user_uuid)
data = await request.get_json(silent=True)
if not isinstance(data, dict):
return jsonify({"error": "JSON object required"}), 400
query = data.get("query")
if not isinstance(query, str) or not query.strip():
return jsonify({"error": "query must be a non-empty string"}), 400
conversation_id = data.get("conversation_id")
if not isinstance(conversation_id, str):
return jsonify({"error": "conversation not found"}), 404
conversation = await _get_owned_conversation(conversation_id, current_user_uuid)
if conversation is None:
return jsonify({"error": "conversation not found"}), 404
from .run_models import ChatRun
# A durable run owns this checkpoint thread (possibly in another process).
if await ChatRun.filter(active_key=str(conversation.id)).exists():
return jsonify({"error": "A run is already active in this conversation"}), 409
await add_message_to_conversation(
conversation=conversation,
message=query,
speaker="user",
user=user,
)
with time_response(stream=False):
async with runtime.thread_lock(str(conversation.id)):
turn = await runtime.prepare_agent_turn(user, query, str(conversation.id))
response = await runtime.invoke_prepared(turn)
message = extract_agent_text(response.get("messages", [])[-1].content)
await persist_agent_tool_messages(
conversation, user, response.get("messages", [])
)
await add_message_to_conversation(
conversation=conversation,
message=message,
speaker="simba",
user=user,
)
return jsonify({"response": message})
@conversation_blueprint.post("/upload-image")
@jwt_access_required
async def upload_image():
current_user_uuid = get_jwt_identity()
await blueprints.users.models.User.get(id=current_user_uuid)
files = await request.files
form = await request.form
file = files.get("file")
conversation_id = form.get("conversation_id")
if not file or not conversation_id:
return jsonify({"error": "file and conversation_id are required"}), 400
file_bytes = file.read()
content_type = file.content_type or "image/jpeg"
try:
processed_bytes, output_content_type = await asyncio.to_thread(
process_image, file_bytes, content_type
)
except ImageValidationError as e:
return jsonify({"error": str(e)}), 400
ext = output_content_type.split("/")[-1]
if ext == "jpeg":
ext = "jpg"
key = f"conversations/{conversation_id}/{uuid.uuid4()}.{ext}"
await s3_upload_image(processed_bytes, key, output_content_type)
return jsonify({"image_key": key})
def _file_key_belongs_to(file_key: str, conversation_id) -> bool:
"""Only allow attachments stored under this conversation's prefix."""
return (
isinstance(file_key, str)
and ".." not in file_key
and file_key.startswith(f"conversations/{conversation_id}/files/")
)
async def _load_attached_file(file_key: str) -> tuple[str, str]:
"""Fetch an attached file from S3 and return (original name, extracted text)."""
data, metadata = await s3_get_file(file_key)
name = unquote(metadata.get("filename", "")) or file_key.rsplit("/", 1)[-1]
ext = file_extension(file_key, None) or ""
# PyMuPDF is CPU-bound; keep it off the event loop so other streams don't stall.
return name, await asyncio.to_thread(extract_text, data, ext)
@conversation_blueprint.post("/upload")
@jwt_access_required
async def upload_attachment():
current_user_uuid = get_jwt_identity()
files = await request.files
form = await request.form
file = files.get("file")
conversation_id = form.get("conversation_id")
if not file or not conversation_id:
return jsonify({"error": "file and conversation_id are required"}), 400
conversation = await _get_owned_conversation(conversation_id, current_user_uuid)
if conversation is None:
return jsonify({"error": "conversation not found"}), 404
file_bytes = file.read()
content_type = (file.content_type or "").split(";")[0].strip().lower()
name = os.path.basename(file.filename or "") or "attachment"
if content_type in IMAGE_TYPES:
try:
processed_bytes, output_content_type = await asyncio.to_thread(
process_image, file_bytes, content_type
)
except Exception as e:
return jsonify({"error": f"Could not process image: {e}"}), 400
ext = output_content_type.split("/")[-1]
if ext == "jpeg":
ext = "jpg"
key = f"conversations/{conversation.id}/{uuid.uuid4()}.{ext}"
await s3_upload_image(processed_bytes, key, output_content_type)
return jsonify({"kind": "image", "image_key": key})
ext = file_extension(name, content_type)
if ext is None:
return (
jsonify(
{
"error": f"Unsupported file type: {content_type or name}. "
"Allowed: images, PDF, TXT, MD, CSV"
}
),
415,
)
try:
text = await asyncio.to_thread(extract_text, file_bytes, ext)
except FileExtractionError as e:
return jsonify({"error": str(e)}), 400
if not text.strip():
message = (
"This PDF has no text layer (scanned?), so Simba can't read it yet."
if ext == "pdf"
else "This file is empty, so there's nothing for Simba to read."
)
return jsonify({"error": message}), 422
key = f"conversations/{conversation.id}/files/{uuid.uuid4()}.{ext}"
await s3_upload_file(file_bytes, key, FILE_TYPES[ext], {"filename": quote(name)})
return jsonify({"kind": "file", "file_key": key, "name": name, "chars": len(text)})
@conversation_blueprint.get("/image/<path:image_key>")
@jwt_access_required
async def serve_image(image_key: str):
url = await s3_presigned_url(image_key)
return jsonify({"url": url})
@conversation_blueprint.post("/stream-query")
@jwt_access_required
async def stream_query():
# Compatibility adapter: disconnect detaches; explicit cancellation uses /runs.
from .runs import create_run, replay_response, RunConflict
try:
run, _ = await create_run(
current_app._get_current_object(),
get_jwt_identity(),
await request.get_json(),
)
except RunConflict as exc:
return jsonify({"error": str(exc)}), exc.status
return await replay_response(run, 0)
async def _chat_run_stream(data, user, conversation, save_user, save_answer):
query_text = data.get("query")
image_key = data.get("image_key")
file_key = data.get("file_key")
# An attached file is extracted and appended to the agent's user turn; the
# saved message only records its name.
saved_text = query_text or ""
if file_key:
if not _file_key_belongs_to(file_key, conversation.id):
raise ValueError("invalid file_key")
try:
file_name, file_text = await _load_attached_file(file_key)
block = attachment_block(file_name, file_text)
except Exception as e:
logging.error(f"Failed to load file {file_key}: {e}")
file_name = file_key.rsplit("/", 1)[-1]
block = "[A file was attached but could not be loaded.]"
query_text = f"{query_text or 'The user attached this file.'}\n\n{block}"
saved_text = f"{saved_text}\n\n📎 {file_name}".strip()
await save_user(saved_text, image_key)
# If an image was uploaded, attach it to the agent request directly
image_block = None
if image_key:
try:
image_bytes, _ = await s3_get_image(image_key)
image_block = await asyncio.to_thread(image_content_block, image_bytes)
except Exception as e:
logging.error(f"Failed to load image {image_key}: {e}")
query_text = f"{query_text or ''}\n\n[An image was sent but could not be loaded.]".strip()
turn = await runtime.prepare_agent_turn(
user,
saved_text or "",
str(conversation.id),
messages=[_build_user_message(query_text or "", image_block)],
)
async def event_generator():
final_message = None
streamed = ""
# Tag-routing state for <think>…</think> blocks.
in_thinking = False
tag_buf = ""
# Fallback heartbeat for models that emit empty chunks before thinking text.
last_thinking = 0.0
turn_has_content = False
# Timing / observability
t0 = time.monotonic()
llm_start_at: float | None = None
first_content_at: float | None = None
tool_timers: dict[str, float] = {} # run_id -> start time
tool_spans: list[dict] = []
# Last output from the in-flight model call; None between calls.
last_model_activity: float | None = None
model_calls: dict[str, dict] = {} # run_id -> timing / token usage
content_chunks = 0
model_name: str | None = None
cancelled = False
persist_attempted = False
async def persist(text: str) -> None:
# At most one attempt: a save cancelled mid-flight must not be retried
# by the cancellation handler, or the answer could be stored twice.
nonlocal persist_attempted
if persist_attempted:
return
persist_attempted = True
await save_answer(text)
def _log_summary(*, error: str | None = None) -> None:
now = time.monotonic()
model_to_first_content_ms = (
round((first_content_at - llm_start_at) * 1000)
if llm_start_at and first_content_at
else None
)
ttft_ms = (
round((first_content_at - t0) * 1000) if first_content_at else None
)
calls = [_model_call_summary(c) for c in model_calls.values()]
prompt_tokens = [c["prompt_tokens"] for c in calls if c["prompt_tokens"]]
completed_output_tokens = sum(
c["output_tokens"] for c in calls if c["output_tokens"] is not None
)
usage_complete = bool(calls) and all(
c["output_tokens"] is not None for c in calls
)
logging.info(
json.dumps(
{
"event": "stream_request",
"conv": str(conversation.id),
"user": user.username,
"total_ms": round((now - t0) * 1000),
"model_to_first_content_ms": model_to_first_content_ms,
"ttft_ms": ttft_ms,
"tokens": completed_output_tokens if usage_complete else None,
"completed_output_tokens": completed_output_tokens,
"usage_complete": usage_complete,
"content_chunks": content_chunks,
"tools": tool_spans,
"model_calls": calls,
"max_prompt_tokens": max(prompt_tokens, default=None),
"model": model_name,
"cancelled": cancelled,
**({"error": error} if error else {}),
}
)
)
try:
aiter = runtime.stream_prepared(turn).__aiter__()
while True:
# Only a silent model call counts as a stall. A model that is
# thinking (or streaming empty chunks) keeps resetting the clock,
# and each new call starts a fresh one.
timeout = stall_timeout(last_model_activity, time.monotonic())
try:
if timeout is not None:
event = await asyncio.wait_for(
aiter.__anext__(), timeout=timeout
)
else:
event = await aiter.__anext__()
except StopAsyncIteration:
break
except asyncio.TimeoutError:
count_result(stream=True, result="stalled")
observe_response(stream=True, seconds=time.monotonic() - t0)
_log_summary(
error=f"model stalled: no output for {MODEL_STALL_TIMEOUT:.0f}s"
)
yield f"data: {json.dumps({'type': 'error', 'message': 'Simba took too long to respond. Please try again.', 'stalled': True})}\n\n"
return
event_type = event.get("event")
if event_type == "on_tool_start":
tool_timers[event["run_id"]] = time.monotonic()
yield f"data: {json.dumps(tool_start_payload(event))}\n\n"
elif event_type in ("on_tool_end", "on_tool_error"):
now = time.monotonic()
start = tool_timers.pop(event["run_id"], None)
tool_spans.append(
{
"tool": event["name"],
"ms": round((now - start) * 1000) if start else None,
}
)
ref = None
# Persist tool output so later turns keep the fetched context
output = event.get("data", {}).get("output")
if event_type == "on_tool_end" and output is not None:
tool_message = await add_message_to_conversation(
conversation=conversation,
message=tool_result_text(event["name"], output),
speaker="tool",
user=user,
)
try:
artifact = await create_artifact_from_output(
output=output,
tool=event["name"],
user=user,
conversation=conversation,
message=tool_message,
)
except Exception as e:
logging.error(
f"Failed to save artifact from {event['name']}: {e}"
)
artifact = None
if artifact is not None:
ref = artifact_ref(artifact)
status = "error" if event_type == "on_tool_error" else "ok"
end_payload = tool_end_payload(
event, start, ref, now=now, status=status
)
yield f"data: {json.dumps(end_payload)}\n\n"
elif event_type == "on_chat_model_start":
turn_has_content = False
now = time.monotonic()
last_model_activity = now
metadata = event.get("metadata", {})
model_calls[event["run_id"]] = {
"start": now,
"model": metadata.get("ls_model_name") or event.get("name"),
"provider": metadata.get("ls_provider"),
}
if llm_start_at is None:
llm_start_at = now
if model_name is None:
model_name = event.get("metadata", {}).get(
"ls_model_name"
) or event.get("name")
elif event_type == "on_chat_model_end":
last_model_activity = None
call = model_calls.get(event["run_id"])
if call is not None:
call["end"] = time.monotonic()
usage = getattr(
event.get("data", {}).get("output"), "usage_metadata", None
)
if usage:
call["prompt_tokens"] = usage.get("input_tokens")
call["output_tokens"] = usage.get("output_tokens")
# Prompt-cache hits, when the provider reports them.
call["cached_prompt_tokens"] = (
usage.get("input_token_details") or {}
).get("cache_read")
call["reasoning_tokens"] = (
usage.get("output_token_details") or {}
).get("reasoning")
elif event_type == "on_chat_model_stream":
now = time.monotonic()
last_model_activity = now
call = model_calls.get(event.get("run_id"))
if call is not None and "first_chunk" not in call:
call["first_chunk"] = now
chunk = event.get("data", {}).get("chunk")
raw = getattr(chunk, "content", None)
if isinstance(raw, str) and raw:
think_text, content_text, in_thinking, tag_buf = _route_content(
tag_buf, raw, in_thinking
)
if think_text:
yield f"data: {json.dumps({'type': 'thinking_delta', 'delta': think_text})}\n\n"
if content_text:
if first_content_at is None:
first_content_at = time.monotonic()
turn_has_content = True
content_chunks += 1
streamed += content_text
yield f"data: {json.dumps({'type': 'content', 'delta': content_text})}\n\n"
elif not turn_has_content:
# Empty chunk before any content — heartbeat for models
# that don't emit <think> tags at all.
now = time.monotonic()
if now - last_thinking >= 1.0:
last_thinking = now
yield f"data: {json.dumps({'type': 'thinking'})}\n\n"
elif event_type == "on_chain_end":
output = event.get("data", {}).get("output")
if isinstance(output, dict):
msgs = output.get("messages", [])
if msgs:
last_msg = msgs[-1]
content = getattr(last_msg, "content", None)
if isinstance(content, str) and content:
final_message = _THINK_RE.sub("", content).strip()
final_message = final_message or streamed
if final_message:
# The durable worker owns persistence independently of subscribers.
await persist(final_message)
yield f"data: {json.dumps({'type': 'response', 'message': final_message})}\n\n"
else:
yield f"data: {json.dumps({'type': 'error', 'message': 'No response generated'})}\n\n"
except asyncio.CancelledError:
cancelled = True
count_result(stream=True, result="cancelled")
observe_response(stream=True, seconds=time.monotonic() - t0)
_log_summary()
text = final_message or (streamed + (tag_buf if not in_thinking else ""))
if text:
await persist(text)
raise
except Exception as e:
count_result(stream=True, result="error")
observe_response(stream=True, seconds=time.monotonic() - t0)
_log_summary(error=str(e))
yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
return
observe_response(stream=True, seconds=time.monotonic() - t0)
count_result(stream=True, result="success" if final_message else "error")
_log_summary(error=None if final_message else "No response generated")
yield "data: [DONE]\n\n"
async for frame in event_generator():
yield frame
@conversation_blueprint.route("/<conversation_id>")
@jwt_access_required
async def get_conversation(conversation_id: str):
user_uuid = get_jwt_identity()
conversation = await _get_owned_conversation(conversation_id, user_uuid)
if conversation is None:
return jsonify({"error": "conversation not found"}), 404
await conversation.fetch_related("messages")
# Keyed by str so the match doesn't depend on the driver's UUID types.
artifacts_by_message = {
str(a.message_id): a
for a in await Artifact.filter(
conversation_id=conversation.id, message_id__isnull=False
).order_by("created_at")
}
# Manually serialize the conversation with messages
messages = []
for msg in conversation.messages:
item = {
"id": str(msg.id),
"text": msg.text,
"speaker": msg.speaker.value,
"created_at": msg.created_at.isoformat(),
"image_key": msg.image_key,
}
if msg.speaker == Speaker.TOOL:
# Tool messages hold raw fetched context for the LLM; only surface
# their header line (e.g. "[web_search result]") in the UI.
item["text"] = msg.text.split("\n", 1)[0]
tool = tool_name_from_message(msg.text)
artifact = artifacts_by_message.get(str(msg.id))
item["tool"] = tool
item["label"] = tool_label(tool)
item["artifact"] = artifact_ref(artifact) if artifact else None
messages.append(item)
return jsonify(
{
"id": str(conversation.id),
"name": conversation.name,
"messages": messages,
"created_at": conversation.created_at.isoformat(),
"updated_at": conversation.updated_at.isoformat(),
}
)
@conversation_blueprint.post("/")
@jwt_access_required
async def create_conversation():
user_uuid = get_jwt_identity()
user = await blueprints.users.models.User.get(id=user_uuid)
conversation = await Conversation.create(
name="New Conversation",
user=user,
)
serialized_conversation = await PydConversation.from_tortoise_orm(conversation)
return jsonify(serialized_conversation.model_dump())
@conversation_blueprint.get("/")
@jwt_access_required
async def get_all_conversations():
user_uuid = get_jwt_identity()
user = await blueprints.users.models.User.get(id=user_uuid)
query = Conversation.filter(user=user)
search = request.args.get("search", "").strip()
if search:
query = query.filter(name__icontains=search)
query = query.order_by("-updated_at")
# `limit` is optional: when omitted we return the full list (backward
# compatible with any non-frontend callers). `offset` only applies with it.
limit = request.args.get("limit", type=int)
if limit is not None:
offset = request.args.get("offset", default=0, type=int)
query = query.offset(offset).limit(limit)
serialized_conversations = await PydListConversation.from_queryset(query)
return jsonify(serialized_conversations.model_dump())
@conversation_blueprint.patch("/<conversation_id>")
@jwt_access_required
async def rename_conversation(conversation_id: str):
user_uuid = get_jwt_identity()
body = await request.get_json()
name = (body or {}).get("name", "").strip()
if not name:
return jsonify({"error": "name is required"}), 400
conversation = await _get_owned_conversation(conversation_id, user_uuid)
if conversation is None:
return jsonify({"error": "conversation not found"}), 404
conversation.name = name[:255]
await conversation.save()
serialized_conversation = await PydConversation.from_tortoise_orm(conversation)
return jsonify(serialized_conversation.model_dump())
@conversation_blueprint.delete("/<conversation_id>")
@jwt_access_required
async def delete_conversation(conversation_id: str):
user_uuid = get_jwt_identity()
conversation = await _get_owned_conversation(conversation_id, user_uuid)
if conversation is None:
return jsonify({"error": "conversation not found"}), 404
store = getattr(current_app, "conversation_summary_store", None)
if store is not None:
await store.delete(str(user_uuid), str(conversation.id))
thread_id = str(conversation.id)
await conversation.delete()
checkpointer = getattr(
getattr(current_app, "main_agent", None), "checkpointer", None
)
if checkpointer is not None:
await delete_thread_checkpoints(checkpointer, thread_id)
return jsonify({"success": True})