diff --git a/.env.example b/.env.example index dfcc9d8..d991f40 100644 --- a/.env.example +++ b/.env.example @@ -24,7 +24,14 @@ BASE_URL=192.168.1.5:8000 # If set, uses llama-server as the primary LLM backend with OpenAI as fallback LLAMA_SERVER_URL=http://192.168.1.213:8080/v1 LLAMA_MODEL_NAME=llama-3.1-8b-instruct +# Abandon a turn when an in-flight model call produces no output (prefill included) +# for this long. Thinking tokens count as output, so long reasoning is fine. TTFT_TIMEOUT_SECONDS=30 +# Context estimates include system instructions and tool definitions. +# Set the window to the smallest backend context; leave room for generation. +CONTEXT_WINDOW_TOKENS=32768 +PROMPT_BUDGET_TOKENS=16000 +RESPONSE_RESERVE_TOKENS=4096 # Embedding Server Configuration # If set, uses a custom OpenAI-compatible embedding server (e.g. llama-server) @@ -70,6 +77,8 @@ YNAB_SYNC_INTERVAL=900 MEALIE_BASE_URL=http://192.168.1.5:9000 # Get your API token from Mealie's user settings page MEALIE_API_TOKEN=your-mealie-api-token +# Group slug used in recipe page links (/g//r/); defaults to "home" +MEALIE_GROUP_SLUG=home # Email Integration # Email Encryption Key (32-byte URL-safe base64) @@ -167,6 +176,8 @@ OBSIDIAN_DEVICE_NAME=simbarag OBSIDIAN_CONTINUOUS_SYNC=false # Local path to Obsidian vault (where files are synced) OBSIDIAN_VAULT_PATH=/app/data/obsidian +# Vault name for obsidian://open links in the artifact panel (optional; no link if unset) +OBSIDIAN_VAULT_NAME= # LangGraph checkpoint store (SQLite). Must be on a persistent volume. # Both blue and green containers share this volume via docker-compose. diff --git a/blueprints/conversation/__init__.py b/blueprints/conversation/__init__.py index a9ea890..268f9ea 100644 --- a/blueprints/conversation/__init__.py +++ b/blueprints/conversation/__init__.py @@ -5,6 +5,7 @@ import os import re import time import uuid +from urllib.parse import quote, unquote from quart import Blueprint, jsonify, make_response, request from quart_jwt_extended import get_jwt_identity @@ -16,9 +17,19 @@ from blueprints.users.decorators import jwt_access_required from metrics import count_result, observe_response, time_response from utils.image_process import image_content_block from utils.strip_markdown import extract_agent_text +from utils.file_extract import ( + FILE_TYPES, + FileExtractionError, + attachment_block, + extract_text, + file_extension, +) +from utils.image_upload import ALLOWED_TYPES as IMAGE_TYPES from utils.image_upload import ImageValidationError, process_image from utils.s3_client import generate_presigned_url as s3_presigned_url +from utils.s3_client import get_file as s3_get_file from utils.s3_client import get_image as s3_get_image +from utils.s3_client import upload_file as s3_upload_file from utils.s3_client import upload_image as s3_upload_image from .logic import ( @@ -37,7 +48,35 @@ from .models import ( from .tool_events import tool_end_payload, tool_name_from_message, tool_start_payload from .tool_labels import tool_label -TTFT_TIMEOUT = float(os.getenv("TTFT_TIMEOUT_SECONDS", "30")) +# Longest an in-flight model call may go without producing any output (prompt +# prefill included) before the turn is abandoned as stalled. Thinking tokens and +# empty chunks count as output; time spent running tools does not count at all. +MODEL_STALL_TIMEOUT = float(os.getenv("TTFT_TIMEOUT_SECONDS", "30")) + + +def stall_timeout(last_model_activity: float | None, now: float) -> float | None: + """Seconds left before the in-flight model call counts as stalled. + + None when no model call is in flight (e.g. while tools run): the httpx read + timeout and the tools' own timeouts cover those phases. + """ + if last_model_activity is None: + return None + return max(MODEL_STALL_TIMEOUT - (now - last_model_activity), 1.0) + + +def _model_call_summary(call: dict) -> dict: + """Per-LLM-call timing and token counts for the stream_request log line.""" + start = call["start"] + first_chunk = call.get("first_chunk") + end = call.get("end") + return { + "prefill_ms": round((first_chunk - start) * 1000) if first_chunk else None, + "ms": round((end - start) * 1000) if end else None, + "prompt_tokens": call.get("prompt_tokens"), + "output_tokens": call.get("output_tokens"), + } + _THINK_OPEN = "" _THINK_CLOSE = "" @@ -203,6 +242,87 @@ async def upload_image(): return jsonify({"image_key": key}) +def _file_key_belongs_to(file_key: str, conversation_id) -> bool: + """Only allow attachments stored under this conversation's prefix.""" + return ( + isinstance(file_key, str) + and ".." not in file_key + and file_key.startswith(f"conversations/{conversation_id}/files/") + ) + + +async def _load_attached_file(file_key: str) -> tuple[str, str]: + """Fetch an attached file from S3 and return (original name, extracted text).""" + data, metadata = await s3_get_file(file_key) + name = unquote(metadata.get("filename", "")) or file_key.rsplit("/", 1)[-1] + ext = file_extension(file_key, None) or "" + # PyMuPDF is CPU-bound; keep it off the event loop so other streams don't stall. + return name, await asyncio.to_thread(extract_text, data, ext) + + +@conversation_blueprint.post("/upload") +@jwt_access_required +async def upload_attachment(): + current_user_uuid = get_jwt_identity() + files = await request.files + form = await request.form + file = files.get("file") + conversation_id = form.get("conversation_id") + + if not file or not conversation_id: + return jsonify({"error": "file and conversation_id are required"}), 400 + + conversation = await _get_owned_conversation(conversation_id, current_user_uuid) + if conversation is None: + return jsonify({"error": "conversation not found"}), 404 + + file_bytes = file.read() + content_type = (file.content_type or "").split(";")[0].strip().lower() + name = os.path.basename(file.filename or "") or "attachment" + + if content_type in IMAGE_TYPES: + try: + processed_bytes, output_content_type = process_image( + file_bytes, content_type + ) + except Exception as e: + return jsonify({"error": f"Could not process image: {e}"}), 400 + ext = output_content_type.split("/")[-1] + if ext == "jpeg": + ext = "jpg" + key = f"conversations/{conversation.id}/{uuid.uuid4()}.{ext}" + await s3_upload_image(processed_bytes, key, output_content_type) + return jsonify({"kind": "image", "image_key": key}) + + ext = file_extension(name, content_type) + if ext is None: + return ( + jsonify( + { + "error": f"Unsupported file type: {content_type or name}. " + "Allowed: images, PDF, TXT, MD, CSV" + } + ), + 415, + ) + + try: + text = await asyncio.to_thread(extract_text, file_bytes, ext) + except FileExtractionError as e: + return jsonify({"error": str(e)}), 400 + if not text.strip(): + message = ( + "This PDF has no text layer (scanned?), so Simba can't read it yet." + if ext == "pdf" + else "This file is empty, so there's nothing for Simba to read." + ) + return jsonify({"error": message}), 422 + + key = f"conversations/{conversation.id}/files/{uuid.uuid4()}.{ext}" + await s3_upload_file(file_bytes, key, FILE_TYPES[ext], {"filename": quote(name)}) + return jsonify({"kind": "file", "file_key": key, "name": name, "chars": len(text)}) + + @conversation_blueprint.get("/image/") @jwt_access_required async def serve_image(image_key: str): @@ -219,12 +339,30 @@ async def stream_query(): query_text = data.get("query") conversation_id = data.get("conversation_id") image_key = data.get("image_key") + file_key = data.get("file_key") conversation = await _get_owned_conversation(conversation_id, current_user_uuid) if conversation is None: return jsonify({"error": "conversation not found"}), 404 + + # An attached file is extracted and appended to the agent's user turn; the + # saved message only records its name. + saved_text = query_text or "" + if file_key: + if not _file_key_belongs_to(file_key, conversation.id): + return jsonify({"error": "invalid file_key"}), 400 + try: + file_name, file_text = await _load_attached_file(file_key) + block = attachment_block(file_name, file_text) + except Exception as e: + logging.error(f"Failed to load file {file_key}: {e}") + file_name = file_key.rsplit("/", 1)[-1] + block = "[A file was attached but could not be loaded.]" + query_text = f"{query_text or 'The user attached this file.'}\n\n{block}" + saved_text = f"{saved_text}\n\n📎 {file_name}".strip() + await add_message_to_conversation( conversation=conversation, - message=query_text or "", + message=saved_text, speaker="user", user=user, image_key=image_key, @@ -271,6 +409,9 @@ async def stream_query(): first_content_at: float | None = None tool_timers: dict[str, float] = {} # run_id -> start time tool_spans: list[dict] = [] + # Last output from the in-flight model call; None between calls. + last_model_activity: float | None = None + model_calls: dict[str, dict] = {} # run_id -> timing / token usage n_tokens = 0 model_name: str | None = None cancelled = False @@ -293,6 +434,8 @@ async def stream_query(): ttft_ms = ( round((first_content_at - t0) * 1000) if first_content_at else None ) + calls = [_model_call_summary(c) for c in model_calls.values()] + prompt_tokens = [c["prompt_tokens"] for c in calls if c["prompt_tokens"]] logging.info( json.dumps( { @@ -304,6 +447,8 @@ async def stream_query(): "ttft_ms": ttft_ms, "tokens": n_tokens, "tools": tool_spans, + "model_calls": calls, + "max_prompt_tokens": max(prompt_tokens, default=None), "model": model_name, "cancelled": cancelled, **({"error": error} if error else {}), @@ -316,15 +461,10 @@ async def stream_query(): payload, version="v2", config=agent_config ).__aiter__() while True: - # Apply a short TTFT timeout only during the window between - # on_chat_model_start and the first content token. Before the - # LLM call starts (tool phase) and after the first token, let - # the httpx read timeout handle stalls instead. - if llm_start_at is not None and first_content_at is None: - ttft_wait = TTFT_TIMEOUT - (time.monotonic() - llm_start_at) - timeout = max(ttft_wait, 1.0) - else: - timeout = None + # Only a silent model call counts as a stall. A model that is + # thinking (or streaming empty chunks) keeps resetting the clock, + # and each new call starts a fresh one. + timeout = stall_timeout(last_model_activity, time.monotonic()) try: if timeout is not None: @@ -336,7 +476,9 @@ async def stream_query(): except StopAsyncIteration: break except asyncio.TimeoutError: - _log_summary(error=f"TTFT timeout after {TTFT_TIMEOUT:.0f}s") + _log_summary( + error=f"model stalled: no output for {MODEL_STALL_TIMEOUT:.0f}s" + ) yield f"data: {json.dumps({'type': 'error', 'message': 'Simba took too long to respond. Please try again.', 'stalled': True})}\n\n" return @@ -388,14 +530,34 @@ async def stream_query(): elif event_type == "on_chat_model_start": turn_has_content = False + now = time.monotonic() + last_model_activity = now + model_calls[event["run_id"]] = {"start": now} if llm_start_at is None: - llm_start_at = time.monotonic() + llm_start_at = now if model_name is None: model_name = event.get("metadata", {}).get( "ls_model_name" ) or event.get("name") + elif event_type == "on_chat_model_end": + last_model_activity = None + call = model_calls.get(event["run_id"]) + if call is not None: + call["end"] = time.monotonic() + usage = getattr( + event.get("data", {}).get("output"), "usage_metadata", None + ) + if usage: + call["prompt_tokens"] = usage.get("input_tokens") + call["output_tokens"] = usage.get("output_tokens") + elif event_type == "on_chat_model_stream": + now = time.monotonic() + last_model_activity = now + call = model_calls.get(event.get("run_id")) + if call is not None and "first_chunk" not in call: + call["first_chunk"] = now chunk = event.get("data", {}).get("chunk") raw = getattr(chunk, "content", None) if isinstance(raw, str) and raw: @@ -457,7 +619,7 @@ async def stream_query(): _log_summary() yield "data: [DONE]\n\n" - return await make_response( + response = await make_response( event_generator(), 200, { @@ -466,6 +628,10 @@ async def stream_query(): "X-Accel-Buffering": "no", }, ) + # Quart's default 60s response deadline cancels even active SSE streams. + # Silent model calls are bounded separately by MODEL_STALL_TIMEOUT. + response.timeout = None + return response @conversation_blueprint.route("/") diff --git a/blueprints/conversation/agents.py b/blueprints/conversation/agents.py index e8d40c7..445077c 100644 --- a/blueprints/conversation/agents.py +++ b/blueprints/conversation/agents.py @@ -1,4 +1,5 @@ import os +from functools import partial from typing import cast import httpx @@ -8,9 +9,24 @@ from langchain.tools import tool from langgraph.prebuilt import create_react_agent from langchain_core.messages import SystemMessage from langchain_core.runnables import RunnableConfig, RunnableLambda +from langchain_core.utils.function_calling import convert_to_openai_tool from langchain_openai import ChatOpenAI from tavily import AsyncTavilyClient +from blueprints.conversation.artifacts_util import ( + NOTE_MAX_CHARS, + format_date_range, + format_day, + json_safe, + make_artifact, + mealie_source_ref, + note_title, + obsidian_source_ref, + search_documents_artifact, + truncate, + unique, +) +from blueprints.conversation.context import RESPONSE_RESERVE_TOKENS, build_context from blueprints.conversation.memory import save_memory from blueprints.conversation.prompts import SIMBA_SYSTEM_PROMPT from blueprints.rag.logic import query_vector_store @@ -41,11 +57,14 @@ if llama_url: timeout=httpx.Timeout(LLAMA_READ_TIMEOUT, connect=10.0), max_retries=1, model_kwargs={"reasoning_effort": "low"}, + # Ask llama-server for token usage on the final chunk (prompt size logging). + stream_usage=True, + max_tokens=RESPONSE_RESERVE_TOKENS, ) else: llama_chat = None -openai_fallback = ChatOpenAI(model="gpt-5-mini") +openai_fallback = ChatOpenAI(model="gpt-5-mini", max_tokens=RESPONSE_RESERVE_TOKENS) model_with_fallback = cast( BaseChatModel, llama_chat.with_fallbacks([openai_fallback]) if llama_chat else openai_fallback, @@ -131,20 +150,36 @@ async def simba_search(query: str): print(f"[SIMBA SEARCH] Found {len(docs)} documents") print(f"[SIMBA SEARCH] Serialized result length: {len(serialized)}") print(f"[SIMBA SEARCH] First 200 chars: {serialized[:200]}") - return serialized, docs + return serialized, search_documents_artifact(query, "paperless", docs) -@tool -async def mealie_todays_meals() -> str: +MEALIE_NOT_CONFIGURED = "Mealie integration is not configured. Please set MEALIE_BASE_URL and MEALIE_API_TOKEN environment variables." + + +def _meal_entry(meal: dict) -> dict: + """Meal plan entry in the `mealie.meal_plan` artifact shape.""" + entry = { + "entry_type": meal["entry_type"], + "name": meal["recipe_name"] or meal["title"] or "Untitled", + } + if meal.get("recipe_slug"): + entry["recipe_slug"] = meal["recipe_slug"] + if meal.get("note"): + entry["note"] = meal["note"] + return entry + + +@tool(response_format="content_and_artifact") +async def mealie_todays_meals(): """Get today's planned meals from Mealie.""" if not mealie_enabled: - return "Mealie integration is not configured. Please set MEALIE_BASE_URL and MEALIE_API_TOKEN environment variables." + return MEALIE_NOT_CONFIGURED, None try: result = await mealie_service.get_todays_meals() if result["total_meals"] == 0: - return f"No meals planned for today ({result['today']})." + return f"No meals planned for today ({result['today']}).", None lines = [f"Meals planned for today ({result['today']}):"] for meal in result["meals"]: @@ -152,13 +187,29 @@ async def mealie_todays_meals() -> str: lines.append(f"- {meal['entry_type'].capitalize()}: {name}") if meal["note"]: lines.append(f" Note: {meal['note']}") - return "\n".join(lines) + + today = result["today"] + artifact = make_artifact( + "mealie.meal_plan", + f"Today's meals · {format_day(today)}", + { + "start_date": today, + "end_date": today, + "days": [ + { + "date": today, + "meals": [_meal_entry(m) for m in result["meals"]], + } + ], + }, + ) + return "\n".join(lines), artifact except Exception as e: - return f"Error fetching today's meals: {str(e)}" + return f"Error fetching today's meals: {str(e)}", None -@tool -async def mealie_meal_plans(start_date: str = "", end_date: str = "") -> str: +@tool(response_format="content_and_artifact") +async def mealie_meal_plans(start_date: str = "", end_date: str = ""): """Get Mealie meal plans for a date range. Args: @@ -166,7 +217,7 @@ async def mealie_meal_plans(start_date: str = "", end_date: str = "") -> str: end_date: YYYY-MM-DD (default: 7 days out) """ if not mealie_enabled: - return "Mealie integration is not configured. Please set MEALIE_BASE_URL and MEALIE_API_TOKEN environment variables." + return MEALIE_NOT_CONFIGURED, None try: result = await mealie_service.get_meal_plans( @@ -175,11 +226,15 @@ async def mealie_meal_plans(start_date: str = "", end_date: str = "") -> str: ) if result["total_plans"] == 0: - return f"No meal plans found from {result['start_date']} to {result['end_date']}." + return ( + f"No meal plans found from {result['start_date']} to {result['end_date']}.", + None, + ) lines = [ f"Meal plans from {result['start_date']} to {result['end_date']} ({result['total_plans']} meals):" ] + days = [] for date, plans in sorted(result["plans_by_date"].items()): lines.append(f"\n{date}:") for plan in plans: @@ -187,20 +242,31 @@ async def mealie_meal_plans(start_date: str = "", end_date: str = "") -> str: lines.append(f" - {plan['entry_type'].capitalize()}: {name}") if plan["note"]: lines.append(f" Note: {plan['note']}") - return "\n".join(lines) + days.append({"date": date, "meals": [_meal_entry(p) for p in plans]}) + + artifact = make_artifact( + "mealie.meal_plan", + f"Meal plan · {format_date_range(result['start_date'], result['end_date'])}", + { + "start_date": result["start_date"], + "end_date": result["end_date"], + "days": days, + }, + ) + return "\n".join(lines), artifact except Exception as e: - return f"Error fetching meal plans: {str(e)}" + return f"Error fetching meal plans: {str(e)}", None -@tool -async def mealie_get_recipe(recipe_slug: str) -> str: +@tool(response_format="content_and_artifact") +async def mealie_get_recipe(recipe_slug: str): """Get full recipe details (ingredients, instructions, times) from Mealie. Args: recipe_slug: Recipe slug from meal plan data """ if not mealie_enabled: - return "Mealie integration is not configured. Please set MEALIE_BASE_URL and MEALIE_API_TOKEN environment variables." + return MEALIE_NOT_CONFIGURED, None try: recipe = await mealie_service.get_recipe(recipe_slug) @@ -236,22 +302,49 @@ async def mealie_get_recipe(recipe_slug: str) -> str: if recipe["tags"]: lines.append(f"\nTags: {', '.join(recipe['tags'])}") - return "\n".join(lines) + instructions = [] + for step in recipe["instructions"]: + instruction = {"text": step.get("text", "")} + if step.get("title"): + instruction["title"] = step["title"] + instructions.append(instruction) + + slug = recipe.get("slug") or recipe_slug + artifact = make_artifact( + "mealie.recipe", + recipe["name"], + json_safe( + { + "slug": slug, + "name": recipe["name"], + "description": recipe["description"], + "servings": recipe["servings"], + "prep_time": recipe["prep_time"], + "cook_time": recipe["cook_time"], + "total_time": recipe["total_time"], + "ingredients": [ing["display"] for ing in recipe["ingredients"]], + "instructions": instructions, + "tags": recipe["tags"], + } + ), + mealie_source_ref(slug), + ) + return "\n".join(lines), artifact except Exception as e: - return f"Error fetching recipe: {str(e)}" + return f"Error fetching recipe: {str(e)}", None -@tool -async def mealie_shopping_lists() -> str: +@tool(response_format="content_and_artifact") +async def mealie_shopping_lists(): """Get Mealie shopping lists with needed and checked-off items.""" if not mealie_enabled: - return "Mealie integration is not configured. Please set MEALIE_BASE_URL and MEALIE_API_TOKEN environment variables." + return MEALIE_NOT_CONFIGURED, None try: result = await mealie_service.get_shopping_lists() if result["total_lists"] == 0: - return "No shopping lists found." + return "No shopping lists found.", None lines = [f"Shopping lists ({result['total_unchecked_items']} items needed):"] for sl in result["shopping_lists"]: @@ -270,9 +363,37 @@ async def mealie_shopping_lists() -> str: lines.append( f" ({len(sl['checked_items'])} items already checked off)" ) - return "\n".join(lines) + + lists = [ + { + "name": sl["name"], + "items": [ + { + "name": item["name"], + "quantity": item["quantity"], + "checked": False, + } + for item in sl["unchecked_items"] + ] + + [ + { + "name": item["name"], + "quantity": item["quantity"], + "checked": True, + } + for item in sl["checked_items"] + ], + } + for sl in result["shopping_lists"] + ] + artifact = make_artifact( + "mealie.shopping_lists", + "Shopping lists", + json_safe({"lists": lists}), + ) + return "\n".join(lines), artifact except Exception as e: - return f"Error fetching shopping lists: {str(e)}" + return f"Error fetching shopping lists: {str(e)}", None @tool @@ -309,34 +430,63 @@ async def mealie_create_meal_plan( return f"Error creating meal plan: {str(e)}" -@tool -async def obsidian_search_notes(query: str) -> str: +OBSIDIAN_NOT_CONFIGURED = "Obsidian integration is not configured. Please set OBSIDIAN_VAULT_PATH environment variable." + + +def _note_artifact(path: str, parsed: dict, title: str | None = None) -> dict: + """`obsidian.note` artifact from ObsidianService.parse_markdown output.""" + frontmatter = json_safe(parsed["metadata"]) + title = title or note_title(path, frontmatter) + return make_artifact( + "obsidian.note", + title, + { + "path": path, + "title": title, + "markdown": truncate(parsed["content"], NOTE_MAX_CHARS), + "tags": unique(parsed["tags"]), + "frontmatter": frontmatter, + }, + obsidian_source_ref(path), + ) + + +@tool(response_format="content_and_artifact") +async def obsidian_search_notes(query: str): """Search personal Obsidian vault notes. Args: query: Search query """ if not obsidian_enabled: - return "Obsidian integration is not configured. Please set OBSIDIAN_VAULT_PATH environment variable." + return OBSIDIAN_NOT_CONFIGURED, None try: # Query vector store filtered to obsidian source only serialized, docs = await query_vector_store(query=query, source="obsidian") - return serialized + return serialized, search_documents_artifact(query, "obsidian", docs) except Exception as e: - return f"Error searching Obsidian notes: {str(e)}" + return f"Error searching Obsidian notes: {str(e)}", None -@tool -async def obsidian_read_note(relative_path: str) -> str: - """Read the full content of a specific Obsidian note by path. +@tool(response_format="content_and_artifact") +async def obsidian_read_note( + relative_path: str, start_line: int = 1, max_lines: int | None = None +): + """Read an Obsidian note, optionally selecting a small range of content lines. + + Use line ranges to retrieve omitted sections of a large note. Args: relative_path: Path relative to vault root (e.g., "notes/my-note.md") + start_line: First content line to read, starting at 1 (excludes frontmatter). + max_lines: Number of lines to read, or None for all remaining lines. """ if not obsidian_enabled: - return "Obsidian integration is not configured. Please set OBSIDIAN_VAULT_PATH environment variable." + return OBSIDIAN_NOT_CONFIGURED, None + if start_line < 1 or (max_lines is not None and max_lines < 1): + return "start_line and max_lines must be positive integers.", None try: note = obsidian_service.read_note(relative_path) @@ -344,16 +494,37 @@ async def obsidian_read_note(relative_path: str) -> str: result = f"File: {note['path']}\n\n" result += f"Frontmatter:\n{content_data['metadata']}\n\n" - result += f"Content:\n{content_data['content']}\n\n" + content = content_data["content"] + if start_line != 1 or max_lines is not None: + lines = content.splitlines() + if start_line > len(lines): + return ( + f"File: {note['path']} has {len(lines)} content lines; start_line is beyond the end.", + None, + ) + end_line = ( + min(len(lines), start_line - 1 + max_lines) + if max_lines is not None + else len(lines) + ) + content = "\n".join(lines[start_line - 1 : end_line]) + result += ( + f"Content lines {start_line}-{end_line} of {len(lines)}:\n{content}\n\n" + ) + else: + result += f"Content:\n{content}\n\n" result += f"Tags: {', '.join(content_data['tags'])}\n" result += f"Contains {len(content_data['wikilinks'])} wikilinks and {len(content_data['embeds'])} embeds" - return result + return result, _note_artifact(note["path"], content_data) except FileNotFoundError: - return f"Note not found at '{relative_path}'. Please check the path is correct." + return ( + f"Note not found at '{relative_path}'. Please check the path is correct.", + None, + ) except Exception as e: - return f"Error reading note: {str(e)}" + return f"Error reading note: {str(e)}", None @tool @@ -391,19 +562,27 @@ async def obsidian_create_note( return f"Error creating note: {str(e)}" -@tool -def journal_get_today() -> str: +@tool(response_format="content_and_artifact") +def journal_get_today(): """Get today's full daily journal note including tasks and log entries.""" if not obsidian_enabled: - return "Obsidian integration is not configured." + return "Obsidian integration is not configured.", None try: note = obsidian_service.get_daily_note() if not note["found"]: - return f"No daily note found for {note['date']}. Use journal_add_task to create one." - return f"Daily note for {note['date']}:\n\n{note['content']}" + return ( + f"No daily note found for {note['date']}. Use journal_add_task to create one.", + None, + ) + artifact = _note_artifact( + note["path"], + obsidian_service.parse_markdown(note["content"]), + title=f"Daily note · {format_day(note['date'])}", + ) + return f"Daily note for {note['date']}:\n\n{note['content']}", artifact except Exception as e: - return f"Error reading daily note: {str(e)}" + return f"Error reading daily note: {str(e)}", None @tool @@ -950,8 +1129,7 @@ async def beli_search_restaurants( tools.append(beli_search_restaurants) -@RunnableLambda -def _build_prompt(state, config: RunnableConfig = None) -> list: +def _build_prompt(state, config: RunnableConfig = None, *, tool_schemas=()) -> list: """Inject the per-request system prompt, replacing any prior system messages.""" system_prompt = ( (config or {}).get("configurable", {}).get("system_prompt", SIMBA_SYSTEM_PROMPT) @@ -959,10 +1137,7 @@ def _build_prompt(state, config: RunnableConfig = None) -> list: # LangGraph passes the full state dict {"messages": [...]}, not just the list messages = state["messages"] if isinstance(state, dict) else state non_system = [m for m in messages if not isinstance(m, SystemMessage)] - # Keep only the most recent messages to stay within the 32k context window. - # Tool call/result pairs each count as 2 messages, so 12 ≈ 4-6 conversational turns. - non_system = non_system[-12:] - return [SystemMessage(content=system_prompt)] + non_system + return build_context(non_system, system_prompt, list(tool_schemas)) def build_agent(checkpointer=None, extra_tools=()): @@ -971,9 +1146,11 @@ def build_agent(checkpointer=None, extra_tools=()): ``extra_tools`` are appended to the shared tool list, e.g. admin-only Homebox and BookStack tools for the admin agent. """ + agent_tools = tools + list(extra_tools) + tool_schemas = [convert_to_openai_tool(t) for t in agent_tools] return create_react_agent( model=model_with_fallback, - tools=tools + list(extra_tools), - prompt=_build_prompt, + tools=agent_tools, + prompt=RunnableLambda(partial(_build_prompt, tool_schemas=tool_schemas)), checkpointer=checkpointer, ) diff --git a/blueprints/conversation/artifacts_util.py b/blueprints/conversation/artifacts_util.py new file mode 100644 index 0000000..cfe64af --- /dev/null +++ b/blueprints/conversation/artifacts_util.py @@ -0,0 +1,178 @@ +"""Helpers for building ToolArtifact dicts returned by agent tools. + +See docs/cowork-ui-plan.md ("Artifacts") for the contract. Artifacts must be +plain JSON-serialisable dicts: they ride along in ToolMessage.artifact and are +stored in the LangGraph checkpoint. +""" + +import os +from datetime import date, datetime +from decimal import Decimal +from pathlib import Path +from urllib.parse import quote, urlencode + +SNIPPET_MAX_CHARS = 400 +NOTE_MAX_CHARS = 50_000 +ELLIPSIS = "…" + + +def make_artifact( + kind: str, title: str, data: dict, source_ref: dict | None = None +) -> dict: + artifact = {"kind": kind, "title": title, "data": data} + if source_ref: + artifact["source_ref"] = source_ref + return artifact + + +def truncate(text: str, limit: int) -> str: + """Cap text at `limit` chars, ending in an ellipsis when cut.""" + if len(text) <= limit: + return text + return text[: limit - len(ELLIPSIS)].rstrip() + ELLIPSIS + + +def snippet(text: str, limit: int = SNIPPET_MAX_CHARS) -> str: + """Collapse whitespace and cap at the snippet limit.""" + return truncate(" ".join((text or "").split()), limit) + + +def json_safe(value): + """Convert a value (e.g. parsed YAML frontmatter) to JSON-safe types.""" + if isinstance(value, (str, int, float, bool, type(None))): + return value + if isinstance(value, Decimal): + return float(value) + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, (list, tuple, set)): + return [json_safe(v) for v in value] + if isinstance(value, dict): + return {str(k): json_safe(v) for k, v in value.items()} + return str(value) + + +def unique(items) -> list: + """Deduplicate while keeping first-seen order.""" + return list(dict.fromkeys(items)) + + +def _parse_date(value: str) -> date | None: + try: + return datetime.strptime(value, "%Y-%m-%d").date() + except (TypeError, ValueError): + return None + + +def format_day(value: str) -> str: + """'2026-10-12' -> 'Oct 12'. Unparseable input is returned unchanged.""" + d = _parse_date(value) + return f"{d:%b} {d.day}" if d else str(value) + + +def format_month(value: str) -> str: + """'2026-10' or '2026-10-01' -> 'October 2026'. Unparseable input is returned unchanged.""" + d = _parse_date(str(value)[:7] + "-01") + return f"{d:%B %Y}" if d else str(value) + + +def format_date_range(start: str, end: str) -> str: + """Human date range: 'Oct 12–19', 'Oct 28–Nov 3', 'Dec 28, 2026–Jan 3, 2027'.""" + s, e = _parse_date(start), _parse_date(end) + if not s or not e: + return f"{start}–{end}" + if s == e: + return f"{s:%b} {s.day}" + if s.year != e.year: + return f"{s:%b} {s.day}, {s.year}–{e:%b} {e.day}, {e.year}" + if s.month == e.month: + return f"{s:%b} {s.day}–{e.day}" + return f"{s:%b} {s.day}–{e:%b} {e.day}" + + +def epoch_to_date(value) -> str | None: + """Epoch seconds -> 'YYYY-MM-DD' (None if not a number).""" + try: + return datetime.fromtimestamp(float(value)).strftime("%Y-%m-%d") + except (TypeError, ValueError, OverflowError, OSError): + return None + + +def mealie_recipe_url(slug: str) -> str | None: + """Recipe page URL for Mealie v2 (`/g//r/`).""" + base_url = os.getenv("MEALIE_BASE_URL", "").rstrip("/") + if not base_url: + return None + group = os.getenv("MEALIE_GROUP_SLUG", "") or "home" + return f"{base_url}/g/{quote(group)}/r/{quote(slug)}" + + +def mealie_source_ref(slug: str) -> dict: + ref = {"system": "mealie", "id": slug} + url = mealie_recipe_url(slug) + if url: + ref["url"] = url + return ref + + +def obsidian_source_ref(path: str) -> dict: + """Obsidian ref; the obsidian:// URL needs OBSIDIAN_VAULT_NAME.""" + ref = {"system": "obsidian", "path": path} + vault = os.getenv("OBSIDIAN_VAULT_NAME", "") + if vault: + ref["url"] = "obsidian://open?" + urlencode( + {"vault": vault, "file": path}, quote_via=quote + ) + return ref + + +def note_title(path: str, frontmatter: dict | None = None) -> str: + """Frontmatter `title` if present, else the filename without extension.""" + title = (frontmatter or {}).get("title") + if isinstance(title, str) and title.strip(): + return title.strip() + return Path(path).stem or path + + +def vault_relative_path(filepath: str) -> str: + """Strip OBSIDIAN_VAULT_PATH from an absolute indexed filepath.""" + vault = os.getenv("OBSIDIAN_VAULT_PATH", "/app/data/obsidian") + try: + return str(Path(filepath).relative_to(vault)) + except ValueError: + return filepath + + +def search_documents_artifact(query: str, source: str, docs) -> dict | None: + """Build a `search.documents` artifact from vector-store Documents.""" + if not docs: + return None + results = [] + for doc in docs: + meta = doc.metadata or {} + result = {"snippet": snippet(doc.page_content)} + if source == "obsidian": + filepath = meta.get("filepath") or "" + path = vault_relative_path(filepath) if filepath else "" + result["title"] = note_title(path, meta) if path else "Untitled note" + if path: + result["path"] = path + created = meta.get("created_at") + if isinstance(created, str) and created: + result["date"] = created[:10] + else: + filename = meta.get("filename") or "" + result["title"] = Path(filename).stem or filename or "Document" + if filename: + result["filename"] = filename + if meta.get("document_type"): + result["document_type"] = str(meta["document_type"]) + created = epoch_to_date(meta.get("created_date")) + if created: + result["date"] = created + results.append(result) + return make_artifact( + "search.documents", + f"Search: {query}", + {"query": query, "source": source, "results": results}, + ) diff --git a/blueprints/conversation/context.py b/blueprints/conversation/context.py new file mode 100644 index 0000000..9babe95 --- /dev/null +++ b/blueprints/conversation/context.py @@ -0,0 +1,286 @@ +"""Budget the model's view of history without changing checkpointed messages. + +Token estimates use UTF-8 bytes / 3 plus message overhead. They are deliberately +more conservative than the usual chars / 4 heuristic, but are not a model +specific tokenizer. A safety reserve covers estimation and chat-template costs. +""" + +import json +import logging +import math +import os + +from dotenv import load_dotenv +from langchain_core.messages import ( + AIMessage, + BaseMessage, + HumanMessage, + SystemMessage, + ToolMessage, +) + +load_dotenv() + +CONTEXT_WINDOW_TOKENS = int(os.getenv("CONTEXT_WINDOW_TOKENS", "32768")) +PROMPT_BUDGET_TOKENS = int(os.getenv("PROMPT_BUDGET_TOKENS", "16000")) +RESPONSE_RESERVE_TOKENS = int(os.getenv("RESPONSE_RESERVE_TOKENS", "4096")) +SAFETY_RESERVE_TOKENS = 2048 +EARLIER_MESSAGE_MAX_CHARS = 4000 +TOOL_RESULT_MAX_CHARS = 12000 +TRUNCATION_NOTE = "\n[truncated: earlier context]" +IMAGE_PLACEHOLDER = "[image from an earlier message]" + + +def estimate_tokens(value) -> int: + text = ( + value + if isinstance(value, str) + else json.dumps(value, ensure_ascii=False, default=str) + ) + return math.ceil(len(text.encode("utf-8")) / 3) + + +def message_tokens(message: BaseMessage) -> int: + # Count tool arguments but never artifact payloads (not sent to the model). + content = message.content + images = 0 + if isinstance(content, list): + images = sum( + isinstance(p, dict) and p.get("type") in ("image", "image_url") + for p in content + ) + content = [ + p + for p in content + if not (isinstance(p, dict) and p.get("type") in ("image", "image_url")) + ] + return ( + 16 + + estimate_tokens(content) + + images * 4096 + + estimate_tokens(getattr(message, "tool_calls", [])) + ) + + +def _cap(text: str, limit: int, note: str) -> str: + if len(text) <= limit: + return text + # Retain both ends: identifiers and totals often appear at the end of results. + head = limit * 3 // 4 + return text[:head] + note + (text[-(limit - head) :] if limit > head else "") + + +def _shrink(message: BaseMessage, limit: int, *, earlier: bool) -> BaseMessage: + note = ( + TRUNCATION_NOTE + if earlier + else "\n[tool result excerpt; omitted content can be fetched again with the source tool]\n" + ) + content = message.content + if isinstance(content, str): + new_content = _cap(content, limit, note) + else: + new_content = [] + remaining = limit + for part in content: + if ( + isinstance(part, dict) + and part.get("type") in ("image_url", "image") + and earlier + ): + new_content.append({"type": "text", "text": IMAGE_PLACEHOLDER}) + elif isinstance(part, dict) and isinstance(part.get("text"), str): + text = _cap(part["text"], remaining, note) + new_content.append({**part, "text": text}) + remaining = max(0, remaining - min(len(part["text"]), remaining)) + elif isinstance(part, str): + new_content.append(_cap(part, remaining, note)) + remaining = max(0, remaining - min(len(part), remaining)) + else: + new_content.append(part) + return ( + message + if new_content == content + else message.model_copy(update={"content": new_content}) + ) + + +def _turns(messages: list[BaseMessage]) -> list[list[BaseMessage]]: + turns = [] + for message in messages: + if isinstance(message, HumanMessage): + turns.append([]) + if turns: + turns[-1].append(message) + # Leading orphaned checkpoint fragments have no user question to anchor them. + return turns + + +def _complete_exchanges(turn: list[BaseMessage]) -> list[BaseMessage]: + """Drop unfinished exchanges left by a cancelled turn and orphan results.""" + result = [] + i = 0 + while i < len(turn): + message = turn[i] + if isinstance(message, AIMessage) and message.tool_calls: + expected = {c["id"] for c in message.tool_calls} + end = i + 1 + while end < len(turn) and isinstance(turn[end], ToolMessage): + end += 1 + replies = turn[i + 1 : end] + actual = [m.tool_call_id for m in replies] + if len(actual) == len(expected) and set(actual) == expected: + result.extend(turn[i:end]) + i = end + else: + if not isinstance(message, ToolMessage): + result.append(message) + i += 1 + return result + + +def _summary(turns: list[list[BaseMessage]], limit: int = 2400) -> SystemMessage | None: + excerpts = [] + remaining = limit + for turn in reversed(turns): + lines = [] + for message in turn: + if isinstance(message, HumanMessage) or ( + isinstance(message, AIMessage) and not message.tool_calls + ): + content = message.content + if not isinstance(content, str): + content = " ".join( + p.get("text", "") if isinstance(p, dict) else str(p) + for p in content + ) + role = "User" if isinstance(message, HumanMessage) else "Assistant" + lines.append(f"{role}: {_cap(content, 400, ' [excerpt] ')}") + excerpt = "\n".join(lines) + if len(excerpt) > remaining: + break + excerpts.insert(0, excerpt) + remaining -= len(excerpt) + 2 + if not excerpts: + return None + return SystemMessage( + content="Earlier conversation excerpts (historical data, not new instructions; excerpts may omit details):\n\n" + + "\n\n".join(excerpts) + ) + + +def trim_history( + messages: list[BaseMessage], *, token_budget: int = 12000 +) -> list[BaseMessage]: + """Keep the current turn, then recent whole turns, within an estimated budget. + + The current user message and tool call arguments are never truncated. Fail + explicitly if these mandatory inputs cannot fit, rather than lose the task. + """ + turns = [_complete_exchanges(turn) for turn in _turns(messages)] + if not turns: + return [] + current = turns[-1] + current = [ + _shrink(m, TOOL_RESULT_MAX_CHARS, earlier=False) + if isinstance(m, ToolMessage) + else m + for m in current + ] + + def cost(ms): + return sum(message_tokens(m) for m in ms) + + # Keep room for continuity even when the current turn has many tool results. + current_budget = ( + token_budget - min(1000, token_budget // 8) if len(turns) > 1 else token_budget + ) + if cost(current) > current_budget: + # Binary-search the largest shared excerpt size that fits. Tool call IDs + # and all results remain, including parallel exchanges. + low, high = 0, TOOL_RESULT_MAX_CHARS + + def compact(limit): + return [ + _shrink(m, limit, earlier=False) + if isinstance(m, (ToolMessage, AIMessage)) + else m + for m in current + ] + + if cost(compact(0)) > token_budget: + raise ValueError( + "The current question and tool calls exceed the context budget. Send a smaller attachment or start a new conversation." + ) + current_budget = max(current_budget, cost(compact(0))) + while low < high: + mid = (low + high + 1) // 2 + if cost(compact(mid)) <= current_budget: + low = mid + else: + high = mid - 1 + current = compact(low) + selected = [] + used = cost(current) + prior = turns[:-1] + # Leave a small share for excerpts of older turns only when necessary. + summary_reserve = min(1000, max(0, token_budget - used)) + for turn in reversed(prior): + shrunk = [_shrink(m, EARLIER_MESSAGE_MAX_CHARS, earlier=True) for m in turn] + turn_cost = cost(shrunk) + if used + turn_cost > token_budget - summary_reserve: + break + selected.insert(0, shrunk) + used += turn_cost + dropped = prior[: len(prior) - len(selected)] + summary = _summary(dropped) + result = [m for turn in selected for m in turn] + current + if summary and used + message_tokens(summary) <= token_budget: + result.insert(0, summary) + logging.info( + json.dumps( + { + "event": "context_budget", + "budget_tokens": token_budget, + "estimated_tokens": cost(result), + "input_messages": len(messages), + "output_messages": len(result), + "dropped_turns": len(dropped), + "summary_included": bool( + result and isinstance(result[0], SystemMessage) + ), + } + ) + ) + return result + + +def build_context( + messages: list[BaseMessage], system_prompt: str, tool_schemas: list[dict] +) -> list[BaseMessage]: + limit = min( + PROMPT_BUDGET_TOKENS, + CONTEXT_WINDOW_TOKENS - RESPONSE_RESERVE_TOKENS - SAFETY_RESERVE_TOKENS, + ) + system = SystemMessage(content=system_prompt) + budget = limit - message_tokens(system) - estimate_tokens(tool_schemas) + if budget <= 0: + raise ValueError( + "System instructions and tool definitions exceed the configured prompt budget." + ) + history = trim_history(messages, token_budget=budget) + logging.info( + json.dumps( + { + "event": "prompt_budget", + "limit_tokens": limit, + "system_tokens": message_tokens(system), + "tool_schema_tokens": estimate_tokens(tool_schemas), + "estimated_prompt_tokens": message_tokens(system) + + estimate_tokens(tool_schemas) + + sum(message_tokens(m) for m in history), + "response_reserve_tokens": RESPONSE_RESERVE_TOKENS, + } + ) + ) + return [system] + history diff --git a/blueprints/conversation/memory.py b/blueprints/conversation/memory.py index 5db37b5..06d4656 100644 --- a/blueprints/conversation/memory.py +++ b/blueprints/conversation/memory.py @@ -1,5 +1,8 @@ from .models import UserMemory from .prompts import SIMBA_SYSTEM_PROMPT +from .context import estimate_tokens + +MEMORY_BUDGET_TOKENS = 1000 async def get_memories_for_user(user_id: str) -> list[str]: @@ -12,7 +15,16 @@ async def build_system_prompt_with_memories(user_id: str) -> str: memories = await get_memories_for_user(user_id) if not memories: return SIMBA_SYSTEM_PROMPT - block = "\n".join(f"- {m}" for m in memories) + selected = [] + used = 0 + for memory in memories: + cost = estimate_tokens(f"- {memory}\n") + if used + cost <= MEMORY_BUDGET_TOKENS: + selected.append(memory) + used += cost + block = "\n".join(f"- {m}" for m in selected) + if len(selected) < len(memories): + block += "\n[Additional saved memories omitted from this prompt.]" return f"{SIMBA_SYSTEM_PROMPT}\n\nUSER MEMORIES (facts the user has asked you to remember):\n{block}" diff --git a/blueprints/ynab/tools.py b/blueprints/ynab/tools.py index c2272bc..469b5d4 100644 --- a/blueprints/ynab/tools.py +++ b/blueprints/ynab/tools.py @@ -14,6 +14,13 @@ from zoneinfo import ZoneInfo from langchain.tools import tool from tortoise import connections +from blueprints.conversation.artifacts_util import ( + format_date_range, + format_month, + json_safe, + make_artifact, +) + from .sync import ensure_fresh logger = logging.getLogger(__name__) @@ -23,6 +30,8 @@ NOT_SYNCED = "YNAB data is unavailable: the budget has not been synced yet." # Keep in step with TOOL_RESULT_MAX_CHARS in blueprints/conversation/logic.py; # longer results get truncated there anyway and crowd the 32k context. MAX_TRANSACTIONS = 40 +# Rows in the ynab.transactions artifact (the side panel, not the LLM) +ARTIFACT_MAX_TRANSACTIONS = 500 # ynab_recent_changes accepts the names it prints as well as stored types ENTITY_TYPE_ALIASES = { @@ -64,6 +73,19 @@ def ynab_tool_errors(fn): return wrapper +def ynab_artifact_tool(fn): + """For content_and_artifact tools: errors become (text, None), and bare + strings (not-synced, bad input, empty results) get no artifact.""" + wrapped = ynab_tool_errors(fn) + + @functools.wraps(fn) + async def wrapper(*args, **kwargs): + result = await wrapped(*args, **kwargs) + return (result, None) if isinstance(result, str) else result + + return wrapper + + # Every transaction line, with split transactions expanded into their # subtransactions so category/payee filters see the real allocation. LINES_CTE = """ @@ -114,6 +136,11 @@ def _usd(value: Decimal | float | None) -> str: return f"{sign}${abs(value):,.2f}" +def _amount(value: Decimal | float | None) -> float: + """Dollar amount for artifacts (JSON has no Decimal).""" + return float(value or 0) + + def _parse_date(value: str, default: date | None) -> date | None: return date.fromisoformat(value) if value else default @@ -128,9 +155,9 @@ async def _query(sql: str, *params) -> list[dict]: return await connections.get("default").execute_query_dict(sql, list(params)) -@tool -@ynab_tool_errors -async def ynab_budget_summary(month: str = "") -> str: +@tool(response_format="content_and_artifact") +@ynab_artifact_tool +async def ynab_budget_summary(month: str = ""): """Get YNAB budget overview for a month: Ready to Assign, income, assigned, spent, age of money, overspent categories, and net worth. Args: @@ -191,12 +218,29 @@ async def ynab_budget_summary(month: str = "") -> str: ) else: lines.append("No overspent categories.") + net_worth = sum(by_budget.values(), Decimal(0)) lines.append( - f"Net worth: {_usd(sum(by_budget.values(), Decimal(0)))} " + f"Net worth: {_usd(net_worth)} " f"(budget accounts {_usd(by_budget.get(True))}, " f"tracking accounts {_usd(by_budget.get(False))})" ) - return "\n".join(lines) + data = { + "month": f"{start:%Y-%m}", + "ready_to_assign": _amount(m["to_be_budgeted"]), + "income": _amount(m["income"]), + "assigned": _amount(m["budgeted"]), + "spent": _amount(spent[0]["spent"]), + "net_worth": _amount(net_worth), + "overspent": [ + {"name": r["name"], "available": _amount(r["balance"])} for r in overspent + ], + } + if m["age_of_money"] is not None: + data["age_of_money"] = m["age_of_money"] + artifact = make_artifact( + "ynab.summary", f"Budget · {format_month(data['month'])}", json_safe(data) + ) + return "\n".join(lines), artifact @tool @@ -230,8 +274,8 @@ async def ynab_account_balances(include_closed: bool = False) -> str: return "\n".join(lines) + f"\n\nNet worth (open accounts): {_usd(total)}" -@tool -@ynab_tool_errors +@tool(response_format="content_and_artifact") +@ynab_artifact_tool async def ynab_search_transactions( start_date: str = "", end_date: str = "", @@ -242,7 +286,7 @@ async def ynab_search_transactions( min_amount: float = 0, max_amount: float = 0, limit: int = 20, -) -> str: +): """Search YNAB transactions. Split transactions are expanded into their category lines. Args: @@ -305,12 +349,16 @@ async def ynab_search_transactions( return f"No transactions found from {range_text}." limit = min(max(1, limit), MAX_TRANSACTIONS) - rows = await _query( + # One query feeds both the text (first `limit` rows) and the artifact; the + # id tie-break keeps the order stable regardless of LIMIT. + all_rows = await _query( f"""{LINES_CTE} SELECT * FROM lines l WHERE {where} - ORDER BY l.date DESC, abs(l.amount) DESC LIMIT {limit}""", + ORDER BY l.date DESC, abs(l.amount) DESC, l.id + LIMIT {ARTIFACT_MAX_TRANSACTIONS}""", *params, ) + rows = all_rows[:limit] lines = [ f"Found {count} transactions from {range_text}. " f"Outflows {_usd(totals[0]['outflow'])}, inflows {_usd(totals[0]['inflow'])}." @@ -327,11 +375,38 @@ async def ynab_search_transactions( lines.append(line) if count > len(rows): lines.append(f"(Showing {len(rows)} most recent of {count})") - return "\n".join(lines) + + transactions = [] + for r in all_rows: + txn = { + "date": r["date"], + "payee": r["payee_name"] or "Unknown payee", + "category": r["category_name"] or "Uncategorized", + "account": r["account_name"], + "amount": _amount(r["amount"]), + } + if r["memo"]: + txn["memo"] = r["memo"] + transactions.append(txn) + title_range = format_date_range(str(start), str(end)) if start else f"through {end}" + artifact = make_artifact( + "ynab.transactions", + f"Transactions · {title_range}", + json_safe( + { + "start_date": start, + "end_date": end, + "count": count, + "total": _amount(totals[0]["outflow"] + totals[0]["inflow"]), + "transactions": transactions, + } + ), + ) + return "\n".join(lines), artifact -@tool -@ynab_tool_errors +@tool(response_format="content_and_artifact") +@ynab_artifact_tool async def ynab_spending_report( group_by: str = "category", start_date: str = "", @@ -339,7 +414,7 @@ async def ynab_spending_report( category: str = "", payee: str = "", top: int = 15, -) -> str: +): """Total YNAB spending grouped by category, category_group, payee, account, or month. Use group_by='month' for trends over time. Args: @@ -377,7 +452,7 @@ async def ynab_spending_report( JOIN ynab_accounts a ON a.id = l.account_id LEFT JOIN ynab_categories c ON c.id = l.category_id LEFT JOIN ynab_category_groups g ON g.id = c.category_group_id - WHERE {' AND '.join(conditions)} + WHERE {" AND ".join(conditions)} GROUP BY 1 ORDER BY {order}""", *params, ) @@ -392,7 +467,7 @@ async def ynab_spending_report( FROM lines l JOIN ynab_accounts a ON a.id = l.account_id LEFT JOIN ynab_categories c ON c.id = l.category_id - WHERE {' AND '.join(conditions)}""", + WHERE {" AND ".join(conditions)}""", *params, ) outflows, inflows = totals[0]["outflows"], totals[0]["inflows"] @@ -409,12 +484,29 @@ async def ynab_spending_report( lines.append(f"- {len(rows) - len(shown)} others: {_usd(rest)}") if group_by == "month" and len(rows) > 1: lines.append(f"Monthly average: {_usd(outflows / len(rows))}") - return "\n".join(lines) + artifact = make_artifact( + "ynab.spending_report", + f"Spending by {group_by.replace('_', ' ')} · " + f"{format_date_range(str(start), str(end))}", + json_safe( + { + "start_date": start, + "end_date": end, + "group_by": group_by, + "rows": [ + {"key": r["key"], "amount": _amount(r["spent"]), "count": r["n"]} + for r in rows + ], + "total": _amount(outflows), + } + ), + ) + return "\n".join(lines), artifact -@tool -@ynab_tool_errors -async def ynab_category_budget(month: str = "", category_group: str = "") -> str: +@tool(response_format="content_and_artifact") +@ynab_artifact_tool +async def ynab_category_budget(month: str = "", category_group: str = ""): """Get YNAB categories for a month grouped by category group: assigned, activity (negative = spent), available, and goal target. Flags overspending. Args: @@ -447,12 +539,23 @@ async def ynab_category_budget(month: str = "", category_group: str = "") -> str lines = [f"Categories for {start:%Y-%m}:"] current = None + groups = [] for r in rows: if not (r["budgeted"] or r["activity"] or r["balance"]): continue if r["group_name"] != current: current = r["group_name"] lines.append(f"\n{current}") + groups.append({"name": current, "categories": []}) + category = { + "name": r["name"], + "assigned": _amount(r["budgeted"]), + "activity": _amount(r["activity"]), + "available": _amount(r["balance"]), + } + if r["goal_target"]: + category["goal_target"] = _amount(r["goal_target"]) + groups[-1]["categories"].append(category) line = ( f" {r['name']}: assigned {_usd(r['budgeted'])}, " f"activity {_usd(r['activity'])}, available {_usd(r['balance'])}" @@ -462,7 +565,14 @@ async def ynab_category_budget(month: str = "", category_group: str = "") -> str if r["balance"] < 0: line += " [OVERSPENT]" lines.append(line) - return "\n".join(lines) + artifact = None + if groups: + artifact = make_artifact( + "ynab.category_budget", + f"Category budget · {format_month(f'{start:%Y-%m}')}", + json_safe({"month": f"{start:%Y-%m}", "groups": groups}), + ) + return "\n".join(lines), artifact @tool diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 79d5e4a..8c10adc 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -146,6 +146,8 @@ pub fn run() { .title("Simba") .inner_size(1200.0, 820.0) .min_inner_size(420.0, 520.0) + // Let HTML5 drag-and-drop reach the webview (file drop in chat). + .disable_drag_drop_handler() .build()?; build_tray(app.handle())?; register_hotkey(app.handle())?; diff --git a/docker-compose.yml b/docker-compose.yml index ff46df1..59bde2f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ x-raggr-common: &raggr-common - YNAB_SYNC_INTERVAL=${YNAB_SYNC_INTERVAL:-900} - MEALIE_BASE_URL=${MEALIE_BASE_URL} - MEALIE_API_TOKEN=${MEALIE_API_TOKEN} + - MEALIE_GROUP_SLUG=${MEALIE_GROUP_SLUG:-home} - TWILIO_ACCOUNT_SID=${TWILIO_ACCOUNT_SID} - TWILIO_AUTH_TOKEN=${TWILIO_AUTH_TOKEN} - TWILIO_WHATSAPP_NUMBER=${TWILIO_WHATSAPP_NUMBER} @@ -40,6 +41,7 @@ x-raggr-common: &raggr-common - OBSIDIAN_DEVICE_NAME=${OBSIDIAN_DEVICE_NAME} - OBSIDIAN_CONTINUOUS_SYNC=${OBSIDIAN_CONTINUOUS_SYNC:-false} - OBSIDIAN_VAULT_PATH=${OBSIDIAN_VAULT_PATH:-/app/data/obsidian} + - OBSIDIAN_VAULT_NAME=${OBSIDIAN_VAULT_NAME:-} - S3_ENDPOINT_URL=${S3_ENDPOINT_URL} - S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID} - S3_SECRET_ACCESS_KEY=${S3_SECRET_ACCESS_KEY} diff --git a/docs/context-management.md b/docs/context-management.md new file mode 100644 index 0000000..9a60960 --- /dev/null +++ b/docs/context-management.md @@ -0,0 +1,43 @@ +# Conversation context management + +The LangGraph checkpoint retains original messages and artifacts. Each model +call constructs a bounded view; compaction never rewrites the checkpoint. + +The prompt budget includes system instructions, saved memories, tool schemas +(including admin tools), message content and tool arguments. The default target +is 16,000 estimated tokens. A 4,096-token generation limit and 2,048-token safety +reserve also constrain the budget against the configured 32,768-token window. + +Estimates use UTF-8 bytes divided by three, message overhead, and a fixed image +allowance. These are estimates rather than model-specific tokenizer counts. +Compare `prompt_budget.estimated_prompt_tokens` with actual `stream_request` +model-call usage and adjust the target/window for the deployed backend. + +The current user request is always retained. Oversized tool results become +marked excerpts preserving their beginning and end; all tool IDs and arguments +remain. If necessary, intermediate assistant text is also shortened. If the +mandatory request and exchange structure cannot fit, the request fails with an +explicit error rather than silently dropping the question. + +Recent history is selected as complete turns. Older user messages, assistant +answers and tool results are capped; earlier images become placeholders. +Incomplete tool exchanges from cancelled calls and orphan results are removed +from the prompt. A small budget is reserved for bounded question/answer excerpts +from older turns. These excerpts are not an LLM-generated rolling summary and do +not guarantee retention of every historical decision. No extra model call is +added to the request path. Saved memories use a separate 1,000-token estimated +budget, preferring recent entries without deleting omitted memories. + +Obsidian reads accept `start_line` (one-based content line, excluding frontmatter) +and `max_lines`, allowing focused follow-up reads. Full artifact content remains +available to the interface and checkpoint. + +Configuration: + +- `CONTEXT_WINDOW_TOKENS`: smallest supported context window across backends. +- `PROMPT_BUDGET_TOKENS`: prompt target, including fixed instructions and tools. +- `RESPONSE_RESERVE_TOKENS`: maximum generated tokens and reserved answer space. + +`context_budget` logs history estimates, message counts, dropped turns and excerpt +inclusion. `prompt_budget` logs fixed overhead and the complete prompt estimate. +Neither log includes message content. diff --git a/raggr-frontend/src/App.css b/raggr-frontend/src/App.css index 9670d2e..6f368e7 100644 --- a/raggr-frontend/src/App.css +++ b/raggr-frontend/src/App.css @@ -202,3 +202,11 @@ body { box-shadow: 0 1px 3px rgba(0,0,0,0.15); } .toggle-track.checked .toggle-thumb { transform: translateX(16px); } + +@keyframes sheet-in { + from { transform: translateY(100%); } + to { transform: translateY(0); } +} +.animate-sheet-in { + animation: sheet-in 0.25s ease-out; +} diff --git a/raggr-frontend/src/api/artifactService.ts b/raggr-frontend/src/api/artifactService.ts new file mode 100644 index 0000000..3d2c707 --- /dev/null +++ b/raggr-frontend/src/api/artifactService.ts @@ -0,0 +1,36 @@ +import { userService } from "./userService"; +import type { Artifact, ArtifactRef } from "./artifactTypes"; + +/** A list entry from `GET /api/artifacts`: the ref plus where it came from. */ +export type ArtifactListItem = ArtifactRef & { + tool: string; + message_id: string | null; + created_at: string; +}; + +class ArtifactService { + private baseUrl = "/api/artifacts"; + + async listArtifacts(conversationId: string): Promise { + const qs = new URLSearchParams({ conversation_id: conversationId }); + const response = await userService.fetchWithAuth(`${this.baseUrl}?${qs}`); + if (!response.ok) { + throw new Error("Failed to list artifacts"); + } + return await response.json(); + } + + async getArtifact(id: string): Promise { + const response = await userService.fetchWithAuth( + `${this.baseUrl}/${encodeURIComponent(id)}`, + ); + if (!response.ok) { + throw new Error( + response.status === 404 ? "Artifact not found" : "Failed to load artifact", + ); + } + return await response.json(); + } +} + +export const artifactService = new ArtifactService(); diff --git a/raggr-frontend/src/api/artifactTypes.ts b/raggr-frontend/src/api/artifactTypes.ts new file mode 100644 index 0000000..b3db843 --- /dev/null +++ b/raggr-frontend/src/api/artifactTypes.ts @@ -0,0 +1,37 @@ +// Artifact types shared by the activity timeline and the artifact panel. +// Shapes follow the contract in docs/cowork-ui-plan.md. + +export type ArtifactKind = + | "mealie.recipe" + | "mealie.meal_plan" + | "mealie.shopping_lists" + | "obsidian.note" + | "search.documents" + | "ynab.summary" + | "ynab.transactions" + | "ynab.spending_report" + | "ynab.category_budget"; + +/** Lightweight reference carried by `tool_end` events and tool messages. */ +export type ArtifactRef = { + id: string; + kind: ArtifactKind; + title: string; +}; + +export type ArtifactSourceRef = { + system: "mealie" | "obsidian" | "paperless" | "ynab" | "web"; + id?: string; + path?: string; + url?: string; +}; + +/** Full artifact as returned by `GET /api/artifacts/`. */ +export type Artifact = ArtifactRef & { + data: object; + source_ref: ArtifactSourceRef | null; + tool: string; + conversation_id: string; + message_id: string | null; + created_at: string; +}; diff --git a/raggr-frontend/src/api/conversationService.ts b/raggr-frontend/src/api/conversationService.ts index 66bbc98..d47cacb 100644 --- a/raggr-frontend/src/api/conversationService.ts +++ b/raggr-frontend/src/api/conversationService.ts @@ -1,10 +1,29 @@ import { userService } from "./userService"; +import type { ArtifactRef } from "./artifactTypes"; + +export type ToolInput = Record; export type SSEEvent = | { type: "thinking" } | { type: "thinking_delta"; delta: string } - | { type: "tool_start"; tool: string } - | { type: "tool_end"; tool: string } + // v2 servers send run_id/label/input and run_id/label/status/ms/artifact; + // older servers send only `tool`, so everything else is optional here. + | { + type: "tool_start"; + tool: string; + run_id?: string; + label?: string; + input?: ToolInput; + } + | { + type: "tool_end"; + tool: string; + run_id?: string; + label?: string; + status?: "ok" | "error"; + ms?: number; + artifact?: ArtifactRef | null; + } | { type: "content"; delta: string } | { type: "response"; message: string } | { type: "error"; message: string; stalled?: boolean }; @@ -17,6 +36,10 @@ interface Message { speaker: "user" | "simba" | "tool"; created_at: string; image_key?: string | null; + // Tool messages only (v2 servers). + tool?: string; + label?: string; + artifact?: ArtifactRef | null; } interface Conversation { @@ -28,6 +51,12 @@ interface Conversation { user_id?: string; } +export type UploadResult = + | { kind: "image"; image_key: string } + | { kind: "file"; file_key: string; name: string; chars: number }; + +export class UploadError extends Error {} + interface QueryResponse { response: string; } @@ -159,16 +188,16 @@ class ConversationService { return await response.json(); } - async uploadImage( + async uploadAttachment( file: File, conversationId: string, - ): Promise<{ image_key: string }> { + ): Promise { const formData = new FormData(); formData.append("file", file); formData.append("conversation_id", conversationId); const response = await userService.fetchWithAuth( - `${this.conversationBaseUrl}/upload-image`, + `${this.conversationBaseUrl}/upload`, { method: "POST", body: formData, @@ -177,8 +206,8 @@ class ConversationService { ); if (!response.ok) { - const data = await response.json(); - throw new Error(data.error || "Failed to upload image"); + const data = await response.json().catch(() => ({})); + throw new UploadError(data.error || "Failed to upload file"); } return await response.json(); @@ -201,11 +230,15 @@ class ConversationService { onEvent: SSEEventCallback, signal?: AbortSignal, imageKey?: string, + fileKey?: string, ): Promise { const body: Record = { query, conversation_id }; if (imageKey) { body.image_key = imageKey; } + if (fileKey) { + body.file_key = fileKey; + } const response = await userService.fetchWithAuth( `${this.conversationBaseUrl}/stream-query`, diff --git a/raggr-frontend/src/components/ActivityTimeline.tsx b/raggr-frontend/src/components/ActivityTimeline.tsx new file mode 100644 index 0000000..1734fa6 --- /dev/null +++ b/raggr-frontend/src/components/ActivityTimeline.tsx @@ -0,0 +1,242 @@ +import React, { useEffect, useId, useState } from "react"; +import { + BookOpen, + CalendarDays, + Cat, + Check, + ChefHat, + ChevronDown, + Clock, + Globe, + Library, + LoaderCircle, + Mail, + NotebookPen, + Package, + PanelRightOpen, + PawPrint, + Utensils, + Wallet, + Wrench, + X, + type LucideIcon, +} from "lucide-react"; +import { cn } from "../lib/utils"; +import type { ArtifactRef } from "../api/artifactTypes"; +import type { ToolInput } from "../api/conversationService"; +import type { ToolMessage } from "../hooks/useChat"; + +// Checked in order, so longer prefixes (simbakit_) come before shorter ones. +const TOOL_ICONS: [prefix: string, icon: LucideIcon][] = [ + ["mealie", ChefHat], + ["ynab", Wallet], + ["obsidian", NotebookPen], + ["journal", BookOpen], + ["simbakit", PawPrint], + ["simba_search", Cat], + ["web_search", Globe], + ["homebox", Package], + ["bookstack", Library], + ["gmail", Mail], + ["beli", Utensils], + ["get_current_date", Clock], +]; + +// Calendar tools are verb-first (get_calendar_events, create_calendar_event), +// so they match anywhere in the name rather than by prefix. +const iconFor = (tool: string): LucideIcon => + tool.includes("calendar") + ? CalendarDays + : (TOOL_ICONS.find(([prefix]) => tool.startsWith(prefix))?.[1] ?? Wrench); + +const MAX_VALUE_CHARS = 40; + +/** `month: 2026-10 · group_by: category`, with long values clipped. */ +export const summariseInput = (input?: ToolInput): string => + Object.entries(input ?? {}) + .filter(([, value]) => value !== "" && value !== null && value !== undefined) + .map(([key, value]) => { + const text = String(value).replace(/\s+/g, " "); + return `${key}: ${text.length > MAX_VALUE_CHARS ? `${text.slice(0, MAX_VALUE_CHARS)}…` : text}`; + }) + .join(" · "); + +export const formatDuration = (ms: number): string => + ms < 1000 ? `${Math.round(ms)} ms` : `${(ms / 1000).toFixed(1)} s`; + +/** + * Wall-clock span of the block when the client timed it (parallel calls + * overlap, so summing would overcount); otherwise the sum of server + * durations. Restored conversations have neither. + */ +const totalDuration = (rows: ToolMessage[]): number | null => { + const starts = rows.map((r) => r.startedAt); + const ends = rows.map((r) => r.endedAt); + if (starts.every((t) => t != null) && ends.every((t) => t != null)) { + return Math.max(...(ends as number[])) - Math.min(...(starts as number[])); + } + const timed = rows.filter((r) => r.ms != null); + return timed.length ? timed.reduce((sum, r) => sum + (r.ms ?? 0), 0) : null; +}; + +type ActivityTimelineProps = { + rows: ToolMessage[]; + /** True while this block belongs to the in-flight turn. */ + active: boolean; + onOpenArtifact: (ref: ArtifactRef) => void; +}; + +export const ActivityTimeline = React.memo( + ({ rows, active, onOpenArtifact }: ActivityTimelineProps) => { + const working = active || rows.some((r) => r.status === "running"); + // null = follow the turn (open while working, collapsed when done). + const [userExpanded, setUserExpanded] = useState(null); + const listId = useId(); + + // Collapse once the turn finishes, even if the user toggled mid-run. + useEffect(() => { + if (!working) setUserExpanded(null); + }, [working]); + + const expanded = userExpanded ?? working; + const failed = rows.filter((r) => r.status === "error").length; + const total = totalDuration(rows); + + const summary = [ + "Simba worked", + `${rows.length} ${rows.length === 1 ? "step" : "steps"}`, + total != null ? formatDuration(total) : null, + failed ? `${failed} failed` : null, + ] + .filter(Boolean) + .join(" · "); + + return ( +
+
+ + + {expanded && ( +
    + {rows.map((row) => ( + + ))} +
+ )} +
+
+ ); + }, +); + +const TimelineRow = ({ + row, + onOpenArtifact, +}: { + row: ToolMessage; + onOpenArtifact: (ref: ArtifactRef) => void; +}) => { + const Icon = iconFor(row.tool); + const inputSummary = summariseInput(row.input); + const artifact = row.artifact; + + return ( +
  • + + + + +
    +
    {row.label}
    + {inputSummary && ( +
    + {inputSummary} +
    + )} +
    + + {artifact && ( + + )} + + + {row.ms != null && row.status !== "running" && ( + {formatDuration(row.ms)} + )} + {row.status === "running" ? ( + + ) : row.status === "error" ? ( + + ) : ( + + )} + +
  • + ); +}; diff --git a/raggr-frontend/src/components/ChatScreen.tsx b/raggr-frontend/src/components/ChatScreen.tsx index dbc95d9..05add45 100644 --- a/raggr-frontend/src/components/ChatScreen.tsx +++ b/raggr-frontend/src/components/ChatScreen.tsx @@ -1,17 +1,21 @@ import React, { useCallback, useState, useRef } from "react"; -import { useNavigate } from "react-router-dom"; -import { PanelLeftOpen, Menu, X } from "lucide-react"; +import { useMatch, useNavigate } from "react-router-dom"; +import { Library, PanelLeftOpen, Menu, X } from "lucide-react"; import { QuestionBubble } from "./QuestionBubble"; import { AnswerBubble } from "./AnswerBubble"; -import { ToolBubble } from "./ToolBubble"; +import { ActivityTimeline } from "./ActivityTimeline"; import { ThinkingBubble } from "./ThinkingBubble"; import { MessageInput } from "./MessageInput"; +import { DropOverlay } from "./DropOverlay"; import { SidebarContent } from "./SidebarContent"; import { AdminPanel } from "./AdminPanel"; import { ScheduledMessagesPanel } from "./ScheduledMessagesPanel"; +import { ArtifactPanel } from "./artifacts/ArtifactPanel"; +import { AUTO_OPEN_MIN_WIDTH, useArtifactPanel } from "./artifacts/useArtifactPanel"; import { cn } from "../lib/utils"; import { useConversations } from "../hooks/useConversations"; -import { useChat } from "../hooks/useChat"; +import { useChat, type ToolMessage } from "../hooks/useChat"; +import type { ArtifactRef } from "../api/artifactTypes"; import catIcon from "../assets/cat.png"; type ChatScreenProps = { @@ -50,14 +54,42 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => { const onSessionExpired = useCallback(() => setAuthenticated(false), [setAuthenticated]); + const conversationId = useMatch("/c/:conversationId")?.params.conversationId; + const conversationIdRef = useRef(conversationId); + conversationIdRef.current = conversationId; + + const artifactPanel = useArtifactPanel(conversationId); + // Opens an artifact in the panel; the timeline's "Open" chip calls this. + const openArtifact = artifactPanel.open; + + // Auto-open on wide windows; newest wins. Narrower windows rely on the + // timeline chip or the header's Artifacts button. + const handleArtifact = useCallback( + (ref: ArtifactRef, turnConversationId: string) => { + if (turnConversationId !== conversationIdRef.current) return; + if (window.innerWidth < AUTO_OPEN_MIN_WIDTH) return; + openArtifact(ref, { auto: true }); + }, + [openArtifact], + ); + + const toggleArtifacts = useCallback(() => { + const p = artifactPanel; + if (p.isOpen && (p.view === "library" || !p.active)) p.collapse(); + else p.showLibrary(); + }, [artifactPanel]); + const { messages, isLoading, streaming, thinking, messagesLoading, - pendingImage, - setPendingImage, + pendingAttachment, + attachFile, + clearAttachment, + attachmentError, + clearAttachmentError, sendMessage, stopGeneration, error, @@ -71,6 +103,7 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => { refreshConversations, onSessionExpired, scrollToBottom, + onArtifact: handleArtifact, }); // Keep ref in sync for scrollToBottom behavior @@ -112,9 +145,6 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => { [deleteConversation], ); - const handleImageSelect = useCallback((file: File) => setPendingImage(file), [setPendingImage]); - const handleClearImage = useCallback(() => setPendingImage(null), [setPendingImage]); - const handleLogout = () => { localStorage.removeItem("access_token"); localStorage.removeItem("refresh_token"); @@ -194,7 +224,8 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => { {showAdminPanel && setShowAdminPanel(false)} />} {showScheduler && setShowScheduler(false)} />} -
    +
    +
    Simba @@ -205,15 +236,52 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => { asksimba
    - +
    + {conversationId && ( + + )} + +
    + {conversationId && ( +
    +

    + {selectedConversation?.title} +

    + +
    + )} + {messages.length === 0 && !messagesLoading ? (
    @@ -236,9 +304,11 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => { onToggleSimbaMode={handleToggleSimbaMode} isLoading={isLoading} onStop={stopGeneration} - pendingImage={pendingImage} - onImageSelect={handleImageSelect} - onClearImage={handleClearImage} + pendingAttachment={pendingAttachment} + onAttach={attachFile} + onClearAttachment={clearAttachment} + attachmentError={attachmentError} + onDismissAttachmentError={clearAttachmentError} autoFocus />
    @@ -248,6 +318,29 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => {
    {messages.map((msg, index) => { + if (msg.speaker === "tool") { + // Consecutive tool calls render as one timeline block, + // emitted at the first row of the run. + if (index > 0 && messages[index - 1].speaker === "tool") { + return null; + } + let end = index; + while (end < messages.length && messages[end].speaker === "tool") { + end++; + } + const rows = messages.slice(index, end) as ToolMessage[]; + return ( + + ); + } + // When the model has been thinking and content is now streaming, // render ThinkingBubble just before the streaming simba message // so it appears above the answer rather than below it. @@ -259,9 +352,7 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => { msg.speaker === "simba"; const bubble = - msg.speaker === "tool" ? ( - - ) : msg.speaker === "simba" ? ( + msg.speaker === "simba" ? ( ) : ( @@ -291,7 +382,12 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => { /> )} - {((isLoading && !streaming && !thinking) || messagesLoading) && ( + {/* Placeholder dots, unless the activity timeline already shows progress */} + {((isLoading && + !streaming && + !thinking && + messages[messages.length - 1]?.speaker !== "tool") || + messagesLoading) && ( )} @@ -339,15 +435,19 @@ export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => { onToggleSimbaMode={handleToggleSimbaMode} isLoading={isLoading} onStop={stopGeneration} - pendingImage={pendingImage} - onImageSelect={handleImageSelect} - onClearImage={handleClearImage} + pendingAttachment={pendingAttachment} + onAttach={attachFile} + onClearAttachment={clearAttachment} + attachmentError={attachmentError} + onDismissAttachmentError={clearAttachmentError} />
    )}
    + +
    ); }; diff --git a/raggr-frontend/src/components/DropOverlay.tsx b/raggr-frontend/src/components/DropOverlay.tsx new file mode 100644 index 0000000..3b22971 --- /dev/null +++ b/raggr-frontend/src/components/DropOverlay.tsx @@ -0,0 +1,87 @@ +import { useEffect, useRef, useState } from "react"; +import { Paperclip } from "lucide-react"; + +const hasFiles = (e: DragEvent) => + Array.from(e.dataTransfer?.types ?? []).includes("Files"); + +type DropOverlayProps = { + onDrop: (file: File) => void; +}; + +/** + * Listens for file drags anywhere in the window and covers its positioned + * parent (the chat column, beside the artifact panel) with a "Drop to attach" + * target. Below lg the panel is a full-screen sheet, so it covers the window + * and sits above the sheet. Drags without files (text, links) + * are ignored so they keep their normal behaviour. + */ +export const DropOverlay = ({ onDrop }: DropOverlayProps) => { + const [active, setActive] = useState(false); + // dragenter/dragleave fire for every element crossed; count to know when + // the drag has really left the window. + const depthRef = useRef(0); + const onDropRef = useRef(onDrop); + onDropRef.current = onDrop; + + useEffect(() => { + const handleEnter = (e: DragEvent) => { + if (!hasFiles(e)) return; + e.preventDefault(); + depthRef.current += 1; + setActive(true); + }; + const handleOver = (e: DragEvent) => { + if (!hasFiles(e)) return; + // Required for the drop event to fire (and to stop the browser + // navigating to the file). + e.preventDefault(); + if (e.dataTransfer) e.dataTransfer.dropEffect = "copy"; + }; + const handleLeave = (e: DragEvent) => { + if (!hasFiles(e)) return; + depthRef.current = Math.max(0, depthRef.current - 1); + if (depthRef.current === 0) setActive(false); + }; + const handleDrop = (e: DragEvent) => { + if (!hasFiles(e)) return; + e.preventDefault(); + depthRef.current = 0; + setActive(false); + const file = e.dataTransfer?.files?.[0]; + if (file) onDropRef.current(file); + }; + + window.addEventListener("dragenter", handleEnter); + window.addEventListener("dragover", handleOver); + window.addEventListener("dragleave", handleLeave); + window.addEventListener("drop", handleDrop); + return () => { + window.removeEventListener("dragenter", handleEnter); + window.removeEventListener("dragover", handleOver); + window.removeEventListener("dragleave", handleLeave); + window.removeEventListener("drop", handleDrop); + }; + }, []); + + if (!active) return null; + + return ( +
    +
    +
    + +
    +

    + Drop to attach +

    +

    Images, PDF, TXT, MD or CSV · up to 10 MB

    +
    +
    + ); +}; diff --git a/raggr-frontend/src/components/MessageInput.tsx b/raggr-frontend/src/components/MessageInput.tsx index 67883e2..2dd2f43 100644 --- a/raggr-frontend/src/components/MessageInput.tsx +++ b/raggr-frontend/src/components/MessageInput.tsx @@ -1,6 +1,12 @@ import React, { useEffect, useMemo, useRef } from "react"; -import { ArrowUp, ImagePlus, Square, X } from "lucide-react"; +import { ArrowUp, FileSpreadsheet, FileText, Paperclip, Square, X } from "lucide-react"; import { cn } from "../lib/utils"; +import { + ATTACHMENT_ACCEPT, + attachmentKind, + fileExtension, + formatBytes, +} from "../lib/attachments"; import { Textarea } from "./ui/textarea"; type MessageInputProps = { @@ -12,9 +18,11 @@ type MessageInputProps = { query: string; isLoading: boolean; onStop: () => void; - pendingImage: File | null; - onImageSelect: (file: File) => void; - onClearImage: () => void; + pendingAttachment: File | null; + onAttach: (file: File) => void; + onClearAttachment: () => void; + attachmentError: string | null; + onDismissAttachmentError: () => void; autoFocus?: boolean; }; @@ -28,15 +36,23 @@ export const MessageInput = React.memo( onToggleSimbaMode, isLoading, onStop, - pendingImage, - onImageSelect, - onClearImage, + pendingAttachment, + onAttach, + onClearAttachment, + attachmentError, + onDismissAttachmentError, autoFocus, }: MessageInputProps) => { const fileInputRef = useRef(null); const textareaRef = useRef(null); - // Create blob URL once per file, revoke on cleanup + const pendingImage = + pendingAttachment && attachmentKind(pendingAttachment) === "image" + ? pendingAttachment + : null; + const pendingFile = pendingAttachment && !pendingImage ? pendingAttachment : null; + + // Create blob URL once per image, revoke on cleanup const previewUrl = useMemo( () => (pendingImage ? URL.createObjectURL(pendingImage) : null), [pendingImage], @@ -59,7 +75,7 @@ export const MessageInput = React.memo( const handleFileChange = (e: React.ChangeEvent) => { const file = e.target.files?.[0]; if (file) { - onImageSelect(file); + onAttach(file); } // Reset so the same file can be re-selected e.target.value = ""; @@ -69,19 +85,21 @@ export const MessageInput = React.memo( if (isLoading) return; const items = e.clipboardData?.items; if (!items) return; - for (const item of items) { + for (const item of Array.from(items)) { if (item.type.startsWith("image/")) { const file = item.getAsFile(); if (file) { e.preventDefault(); - onImageSelect(file); + onAttach(file); return; } } } }; - const canSend = !isLoading && Boolean(query.trim() || pendingImage); + const canSend = !isLoading && Boolean(query.trim() || pendingAttachment); + const FileIcon = + pendingFile && fileExtension(pendingFile.name) === "csv" ? FileSpreadsheet : FileText; return (
    )} + {/* File chip */} + {pendingFile && ( +
    +
    +
    + +
    +
    +
    + {pendingFile.name} +
    +
    + {fileExtension(pendingFile.name).toUpperCase() || "File"} ·{" "} + {formatBytes(pendingFile.size)} +
    +
    + +
    +
    + )} + + {attachmentError && ( +
    +
    + {attachmentError} + +
    +
    + )} + {/* Textarea */}