Merge pull request 'Add Immich MCP technical design' (#90) from docs/immich-mcp-design into main

Design for an in-repo stdio MCP server that lets Simba (and Claude Code)
find existing albums, summarize trips by date clusters, and create or
fill albums behind a write flag and a confirmed-count check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.env.example

@@ -1,14 +1,15 @@
 # Database Configuration
-# PostgreSQL is recommended (required for OIDC features)
-DATABASE_URL=postgres://raggr:changeme@postgres:5432/raggr
+# PostgreSQL with pgvector is required; production rejects default passwords.
+DATABASE_URL=postgres://raggr:replace-with-random-password@postgres:5432/raggr
 
 # PostgreSQL credentials (if using docker-compose postgres service)
 POSTGRES_USER=raggr
-POSTGRES_PASSWORD=changeme
+POSTGRES_PASSWORD=replace-with-random-password
 POSTGRES_DB=raggr
 
 # JWT Configuration
-JWT_SECRET_KEY=your-secret-key-here
+# Required: generate a random secret with openssl rand -hex 32
+JWT_SECRET_KEY=
 
 # Metrics Configuration
 # Bearer token for the /metrics endpoint (Prometheus scrape + app token gate).
@@ -17,13 +18,15 @@ JWT_SECRET_KEY=your-secret-key-here
 METRICS_TOKEN=
 
 # Paperless Configuration
-PAPERLESS_TOKEN=your-paperless-token
-BASE_URL=192.168.1.5:8000
+PAPERLESS_TOKEN=
+# Current Paperless clients prepend http://; use host[:port] without a scheme.
+BASE_URL=
 
 # llama-server Configuration (OpenAI-compatible API)
 # If set, uses llama-server as the primary LLM backend with OpenAI as fallback
-LLAMA_SERVER_URL=http://192.168.1.213:8080/v1
+LLAMA_SERVER_URL=
 LLAMA_MODEL_NAME=llama-3.1-8b-instruct
+LLAMA_TIMEOUT_SECONDS=120
 # Abandon a turn when an in-flight model call produces no output (prefill included)
 # for this long. Thinking tokens count as output, so long reasoning is fine.
 TTFT_TIMEOUT_SECONDS=30
@@ -32,33 +35,46 @@ TTFT_TIMEOUT_SECONDS=30
 CONTEXT_WINDOW_TOKENS=32768
 PROMPT_BUDGET_TOKENS=16000
 RESPONSE_RESERVE_TOKENS=4096
+# static: bind every role-appropriate tool on each call (stable, cacheable prompt
+# prefix). dynamic: keyword-route tool groups per call with a discovery tool.
+TOOL_ROUTING_MODE=static
 # Optional durable summary sidecar; defaults to CHECKPOINT_DB_PATH + .summaries.db
 # Store on the same persistent shared volume as the checkpoints.
 # CONVERSATION_SUMMARY_DB_PATH=/app/checkpoints/conversation-summaries.db
+# Durable chat runs (see docs/chat-runs.md). Lease = abandoned-worker timeout;
+# max = whole generation; drain = shutdown wait for in-flight generations.
+# scripts/deploy.sh gives the old container drain + 60s (override DEPLOY_STOP_SECONDS).
+CHAT_RUN_LEASE_SECONDS=30
+CHAT_RUN_MAX_SECONDS=1800
+CHAT_RUN_DRAIN_SECONDS=1800
 
 # Embedding Server Configuration
 # If set, uses a custom OpenAI-compatible embedding server (e.g. llama-server)
 # Falls back to OpenAI embeddings if not set
-EMBEDDING_SERVER_URL=http://192.168.1.7:8086/v1
-EMBEDDING_MODEL_NAME=all-minilm
+EMBEDDING_SERVER_URL=
+EMBEDDING_MODEL_NAME=text-embedding-3-small
 
 # OpenAI Configuration
-OPENAI_API_KEY=your-openai-api-key
+# Required even with llama-server: the fallback client is always initialized.
+OPENAI_API_KEY=
 
 # Tavily Configuration (for web search)
-TAVILY_API_KEY=your-tavily-api-key
+# Legacy TAVILIY_API_KEY is accepted by Compose/startup; TAVILY_API_KEY wins.
+# Current agent initialization needs this key even when web search is unused.
+TAVILY_API_KEY=
 
 # Immich Configuration
 IMMICH_URL=http://192.168.1.5:2283
-IMMICH_API_KEY=your-immich-api-key
-SEARCH_QUERY=simba cat
+IMMICH_API_KEY=
+SEARCH_QUERY="simba cat"
 DOWNLOAD_DIR=./simba_photos
 
-# OIDC Configuration (Authelia)
-OIDC_ISSUER=https://auth.example.com
-OIDC_CLIENT_ID=simbarag
-OIDC_CLIENT_SECRET=your-client-secret-here
-OIDC_REDIRECT_URI=http://localhost:8080/
+# Optional OIDC (Authelia): set all four fields together. Production requires HTTPS.
+# Register the public callback URI: https://your-host/api/user/oidc/callback
+OIDC_ISSUER=
+OIDC_CLIENT_ID=
+OIDC_CLIENT_SECRET=
+OIDC_REDIRECT_URI=
 OIDC_USE_DISCOVERY=true
 
 # Optional: Manual OIDC endpoints (if discovery is disabled)
@@ -69,7 +85,7 @@ OIDC_USE_DISCOVERY=true
 
 # YNAB Configuration
 # Get your Personal Access Token from https://app.ynab.com/settings/developer
-YNAB_ACCESS_TOKEN=your-ynab-personal-access-token
+YNAB_ACCESS_TOKEN=
 # Optional: Specify a budget ID, or leave empty to use the default/first budget
 YNAB_BUDGET_ID=
 # Seconds between background YNAB delta syncs. Keep under the agent tools' 120s
@@ -78,21 +94,22 @@ YNAB_SYNC_INTERVAL=90
 
 # Mealie Configuration
 # Base URL for your Mealie instance (e.g., http://192.168.1.5:9000 or https://mealie.example.com)
-MEALIE_BASE_URL=http://192.168.1.5:9000
+MEALIE_BASE_URL=
 # Get your API token from Mealie's user settings page
-MEALIE_API_TOKEN=your-mealie-api-token
+MEALIE_API_TOKEN=
 # Group slug used in recipe page links (/g/<slug>/r/<recipe>); defaults to "home"
 MEALIE_GROUP_SLUG=home
 
 # Email Integration
+# Required even without email: encrypted model fields initialize on startup.
 # Email Encryption Key (32-byte URL-safe base64)
 # Generate with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
-FERNET_KEY=your-fernet-key-here
+FERNET_KEY=
 
 # Twilio Configuration (WhatsApp)
-TWILIO_ACCOUNT_SID=your-twilio-account-sid
-TWILIO_AUTH_TOKEN=your-twilio-auth-token
-TWILIO_WHATSAPP_NUMBER=whatsapp:+14155238886
+TWILIO_ACCOUNT_SID=
+TWILIO_AUTH_TOKEN=
+TWILIO_WHATSAPP_NUMBER=
 # Comma-separated list of WhatsApp numbers allowed to use the service (e.g., whatsapp:+1234567890)
 # Use * to allow any number
 ALLOWED_WHATSAPP_NUMBERS=
@@ -116,10 +133,10 @@ EMAIL_HMAC_SECRET=
 MAILGUN_SIGNATURE_VALIDATION=true
 
 # SendBlue Configuration (iMessage)
-SENDBLUE_API_KEY=your-sendblue-api-key
-SENDBLUE_API_SECRET=your-sendblue-api-secret
-SENDBLUE_FROM_NUMBER=+1XXXXXXXXXX
-SENDBLUE_WEBHOOK_SECRET=your-sendblue-webhook-secret
+SENDBLUE_API_KEY=
+SENDBLUE_API_SECRET=
+SENDBLUE_FROM_NUMBER=
+SENDBLUE_WEBHOOK_SECRET=
 # Set to false to disable SendBlue signature validation in development
 SENDBLUE_SIGNATURE_VALIDATION=true
 # Comma-separated list of iMessage numbers allowed to use the service (E.164 format)
@@ -142,42 +159,47 @@ GOOGLE_TOKEN_FILE=/app/config/google-token.json
 CRON_SECRET=
 # Cronicle server URL and API key (from Cronicle Admin → API Keys)
 # Used to auto-register scheduled jobs on startup.
-CRONICLE_URL=http://cronicle:3012
+CRONICLE_URL=
 CRONICLE_API_KEY=
 
 # SimbaKit / PetKit Configuration
 # PetKit account credentials (used by SimbaKit to poll device data)
-PETKIT_USERNAME=your-petkit-email
-PETKIT_PASSWORD=your-petkit-password
+PETKIT_USERNAME=
+PETKIT_PASSWORD=
 PETKIT_REGION=US
 PETKIT_TIMEZONE=America/New_York
 # Internal URL for SimbaKit service (Docker network)
-SIMBAKIT_URL=http://simbakit:8585
+SIMBAKIT_URL=
 
 # Homebox inventory (admin-only agent tools via homebox-mcp, needs Homebox >= 0.26)
 # API key from Homebox Profile -> API Keys (starts with hb_). Leave empty to disable.
-HOMEBOX_URL=https://homebox.example.com
+HOMEBOX_URL=
 HOMEBOX_TOKEN=
 
 # BookStack wiki (admin-only agent tools via bookstack-mcp: read, create, update; no deletes)
 # API token from BookStack Profile -> API Tokens (user needs the "Access System API" permission).
 # Leave empty to disable.
-BOOKSTACK_BASE_URL=https://bookstack.example.com
+BOOKSTACK_BASE_URL=
 BOOKSTACK_TOKEN_ID=
 BOOKSTACK_TOKEN_SECRET=
 
 # Obsidian Configuration (headless sync)
 # Obsidian account credentials (used for `ob login` on container startup)
-OBSIDIAN_EMAIL=your-obsidian-email
-OBSIDIAN_PASSWORD=your-obsidian-password
+OBSIDIAN_EMAIL=
+OBSIDIAN_PASSWORD=
 # Vault ID to sync (found in Obsidian sync settings)
-OBSIDIAN_VAULT_ID=your-vault-id
+OBSIDIAN_VAULT_ID=
 # End-to-end encryption password (if vault uses E2E encryption)
 OBSIDIAN_E2E_PASSWORD=
 # Device name shown in Obsidian sync activity
 OBSIDIAN_DEVICE_NAME=simbarag
 # Set to true to run continuous sync in the background
 OBSIDIAN_CONTINUOUS_SYNC=false
+# bidirectional: notes and journal tasks Simba writes sync back to your devices.
+# pull-only: download only, Simba's writes stay in the container's copy.
+# mirror-remote: download only and revert local changes.
+OBSIDIAN_SYNC_MODE=bidirectional
+OBSIDIAN_SYNC_INTERVAL=60
 # Local path to Obsidian vault (where files are synced)
 OBSIDIAN_VAULT_PATH=/app/data/obsidian
 # Vault name for obsidian://open links in the artifact panel (optional; no link if unset)
@@ -186,7 +208,21 @@ OBSIDIAN_VAULT_NAME=
 # LangGraph checkpoint store (SQLite). Must be on a persistent volume.
 # Both blue and green containers share this volume via docker-compose.
 CHECKPOINT_DB_PATH=/app/checkpoints/checkpoints.db
+# Checkpoints keep a full message-list snapshot per turn; this bounds how many
+# are kept per conversation thread (older ones are pruned after each turn).
+CHECKPOINT_KEEP_PER_THREAD=20
 
 # Beli restaurant recommendations
 # Beli account credentials for each profile are stored encrypted in the database.
 # No additional env vars needed — auth is done via the Beli API at login time.
+
+# Optional S3 image storage (leave credentials empty to disable)
+S3_ENDPOINT_URL=
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
+S3_BUCKET_NAME=asksimba-images
+S3_REGION=garage
+
+# Ollama is used by image/PDF helper workflows, not the main chat agent.
+OLLAMA_URL=http://localhost:11434
+OLLAMA_HOST=http://localhost:11434

.gitea/workflows/ci.yml

@@ -0,0 +1,51 @@
+name: CI
+on:
+  push:
+  pull_request:
+
+jobs:
+  backend:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.2.2
+      - uses: actions/setup-python@v5.6.0
+        with:
+          python-version: '3.13.2'
+      - name: Install pinned tooling
+        run: python -m pip install uv==0.7.2 ruff==0.8.2
+      - name: Install locked backend and test dependencies
+        run: |
+          uv lock --check
+          uv sync --frozen --extra test
+      - name: Ruff
+        run: ruff check .
+      - name: Unit tests (no service credentials)
+        env:
+          OPENAI_API_KEY: synthetic-unit-test-key
+          TAVILY_API_KEY: synthetic-unit-test-key
+        run: uv run --frozen --extra test pytest tests/unit -v
+      - name: Deterministic context evaluation
+        run: uv run --frozen --extra test python -m scripts.evaluate_context
+
+  frontend:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: raggr-frontend
+    steps:
+      - uses: actions/checkout@v4.2.2
+      # No `cache: yarn` here: setup-node requires yarn to already be on PATH
+      # for that, and the Node it installs doesn't ship one.
+      - uses: actions/setup-node@v4.4.0
+        with:
+          node-version: '22.14.0'
+      - name: Install Yarn (lockfile v1)
+        run: npm install --global yarn@1.22.22
+      - name: Install locked frontend dependencies
+        run: yarn install --frozen-lockfile
+      - name: Renderer tests
+        run: node --test tests/*.test.cjs
+      - name: Typecheck
+        run: yarn tsc --noEmit
+      - name: Build
+        run: yarn build

Dockerfile

@@ -1,16 +1,30 @@
-FROM python:3.13-slim
+FROM node:22.14.0-bookworm-slim AS frontend
+WORKDIR /frontend
+COPY raggr-frontend/package.json raggr-frontend/yarn.lock ./
+RUN yarn install --frozen-lockfile
+# Copy build inputs explicitly: there is no root .dockerignore, and copying the
+# whole directory would overwrite locked Linux deps with host node_modules.
+COPY raggr-frontend/src ./src
+COPY raggr-frontend/public ./public
+COPY raggr-frontend/rsbuild.config.ts raggr-frontend/tsconfig.json raggr-frontend/postcss.config.mjs ./
+RUN yarn build
+
+FROM ghcr.io/astral-sh/uv:0.7.2 AS uv
+
+FROM python:3.13.2-slim-bookworm
 
 WORKDIR /app
 
-# Install system dependencies, Node.js, Yarn, and uv
+# Node is required by the optional stdio MCP and Obsidian tools at runtime.
+COPY --from=frontend /usr/local/bin/node /usr/local/bin/node
+COPY --from=frontend /usr/local/lib/node_modules /usr/local/lib/node_modules
+RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm
+COPY --from=uv /uv /uvx /usr/local/bin/
 RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
-    && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
-    && apt-get install -y nodejs \
-    && npm install -g yarn obsidian-headless @googleworkspace/cli \
-    && rm -rf /var/lib/apt/lists/* \
-    && curl -LsSf https://astral.sh/uv/install.sh | sh
+    && npm install -g obsidian-headless@0.0.8 @googleworkspace/cli@0.8.0 \
+    && rm -rf /var/lib/apt/lists/*
 
 # Add uv to PATH
 ENV PATH="/root/.local/bin:$PATH"
@@ -22,10 +36,15 @@ RUN uv tool install homebox-mcp==1.1.0
 RUN npm install -g bookstack-mcp@5.6.0
 
 # Copy dependency files
-COPY pyproject.toml ./
+COPY pyproject.toml uv.lock ./
 
 # Install Python dependencies using uv
-RUN uv pip install --system -e .
+# Export the frozen graph and install it system-wide: python and aerich remain
+# available to startup.sh and docker compose exec without activating a venv.
+RUN uv lock --check \
+    && uv export --frozen --no-dev --no-emit-project --quiet -o /tmp/requirements.txt \
+    && uv pip install --system --require-hashes -r /tmp/requirements.txt \
+    && rm /tmp/requirements.txt
 
 # Copy application code
 COPY *.py ./
@@ -37,14 +56,10 @@ COPY scripts ./scripts
 COPY startup.sh ./
 RUN chmod +x startup.sh
 
-# Copy frontend code and build
-COPY raggr-frontend ./raggr-frontend
-WORKDIR /app/raggr-frontend
-RUN yarn install && yarn build
-WORKDIR /app
+COPY --from=frontend /frontend/dist ./raggr-frontend/dist
 
 # Create database directory
-RUN mkdir -p /app/database
+RUN mkdir -p /app/database /app/checkpoints
 
 # Expose port
 EXPOSE 8080

Makefile

@@ -69,14 +69,14 @@ migrate-history:
 
 # Tests
 test:
-	pytest tests/ -v
+	OPENAI_API_KEY=synthetic-unit-test-key TAVILY_API_KEY=synthetic-unit-test-key uv run --frozen --extra test pytest tests/unit -v
 
 test-cov:
-	pytest tests/ -v --cov
+	OPENAI_API_KEY=synthetic-unit-test-key TAVILY_API_KEY=synthetic-unit-test-key uv run --frozen --extra test pytest tests/unit -v --cov
 
 # Frontend
 frontend:
-	cd raggr-frontend && yarn install && yarn build
+	cd raggr-frontend && npm ci && npm run build
 
 # Desktop app (Tauri). Point it at the server with SIMBA_URL, e.g.
 #   make desktop-dev SIMBA_URL=https://simba.example.com
@@ -88,3 +88,16 @@ desktop:
 	cd desktop && npm install && SIMBA_URL=$(SIMBA_URL) npx tauri build --bundles app
 	rm -rf /Applications/Simba.app
 	cp -R desktop/src-tauri/target/release/bundle/macos/Simba.app /Applications/
+
+.PHONY: lint evaluate validate-config frontend-check
+lint:
+	uvx ruff==0.8.2 check .
+
+evaluate:
+	uv run --frozen --extra test python -m scripts.evaluate_context
+
+validate-config:
+	uv run --frozen python scripts/validate_config.py
+
+frontend-check:
+	cd raggr-frontend && npm ci && node --test tests/*.test.cjs && npx --no-install tsc --noEmit && npm run build

README.md

@@ -1,370 +1,60 @@
-# SimbaRAG 🐱
+# SimbaRAG
 
-A Retrieval-Augmented Generation (RAG) conversational AI system for querying information about Simba the cat. Built with LangChain, ChromaDB, and modern web technologies.
+SimbaRAG is a personal assistant with a Quart backend and React 19 frontend, document retrieval, persistent conversations, and configurable household integrations.
 
-## Features
+PostgreSQL stores application data and **pgvector** stores document embeddings (`simba_docs`). LangGraph checkpoints and conversation summaries use separate SQLite files on a persistent volume. Chat uses an OpenAI-compatible **llama-server** when configured, with **OpenAI `gpt-5-mini` fallback**; without llama-server it uses OpenAI directly. Embeddings use a configured OpenAI-compatible server or OpenAI `text-embedding-3-small`. Ollama is used by image/PDF helper workflows.
 
-- 🤖 **Intelligent Conversations** - LangChain-powered agent with tool use and memory
-- 📚 **Document Retrieval** - RAG system using ChromaDB vector store
-- 🔍 **Web Search** - Integrated Tavily API for real-time web searches
-- 🔐 **OIDC Authentication** - Secure auth via Authelia with LDAP group support
-- 💬 **Multi-Conversation** - Manage multiple conversation threads per user
-- 🎨 **Modern UI** - React 19 frontend with Tailwind CSS
-- 🐳 **Docker Ready** - Containerized deployment with Docker Compose
+## Setup
 
-## System Architecture
-
-```mermaid
-graph TB
-    subgraph "Client Layer"
-        Browser[Web Browser]
-    end
-
-    subgraph "Frontend - React"
-        UI[React UI<br/>Tailwind CSS]
-        Auth[Auth Service]
-        API[API Client]
-    end
-
-    subgraph "Backend - Quart/Python"
-        App[Quart App<br/>app.py]
-
-        subgraph "Blueprints"
-            Users[Users Blueprint<br/>OIDC + JWT]
-            Conv[Conversation Blueprint<br/>Chat Management]
-            RAG[RAG Blueprint<br/>Document Indexing]
-        end
-
-        Agent[LangChain Agent<br/>main.py]
-        LLM[LLM Client<br/>llm.py]
-    end
-
-    subgraph "Tools & Utilities"
-        Search[Simba Search Tool]
-        Web[Web Search Tool<br/>Tavily]
-    end
-
-    subgraph "Data Layer"
-        Postgres[(PostgreSQL<br/>Users & Conversations)]
-        Chroma[(ChromaDB<br/>Vector Store)]
-    end
-
-    subgraph "External Services"
-        Authelia[Authelia<br/>OIDC Provider]
-        LLDAP[LLDAP<br/>User Directory]
-        Ollama[Ollama<br/>Local LLM]
-        OpenAI[OpenAI<br/>Fallback LLM]
-        Paperless[Paperless-NGX<br/>Documents]
-        TavilyAPI[Tavily API<br/>Web Search]
-    end
-
-    Browser --> UI
-    UI --> Auth
-    UI --> API
-    API --> App
-
-    App --> Users
-    App --> Conv
-    App --> RAG
-
-    Conv --> Agent
-    Agent --> Search
-    Agent --> Web
-    Agent --> LLM
-
-    Search --> Chroma
-    Web --> TavilyAPI
-    RAG --> Chroma
-    RAG --> Paperless
-
-    Users --> Postgres
-    Conv --> Postgres
-
-    Users --> Authelia
-    Authelia --> LLDAP
-
-    LLM --> Ollama
-    LLM -.Fallback.-> OpenAI
-
-    style Browser fill:#e1f5ff
-    style UI fill:#fff3cd
-    style App fill:#d4edda
-    style Agent fill:#d4edda
-    style Postgres fill:#f8d7da
-    style Chroma fill:#f8d7da
-    style Ollama fill:#e2e3e5
-    style OpenAI fill:#e2e3e5
-```
-
-## Quick Start
-
-### Prerequisites
-
-- Docker & Docker Compose
-- PostgreSQL (or use Docker)
-- Ollama (optional, for local LLM)
-- Paperless-NGX instance (for document source)
-
-### Installation
-
-1. **Clone the repository**
-
-```bash
-git clone https://github.com/yourusername/simbarag.git
-cd simbarag
-```
-
-2. **Configure environment variables**
+Install Python 3.13, uv 0.7.2, Node 22.14.0, and Docker Compose v2. From the repository root:
 
 ```bash
 cp .env.example .env
-# Edit .env with your configuration
+# Edit .env before starting services; it is deliberately not deployable as-is.
+uv sync --frozen --extra test
+cd raggr-frontend
+npm ci
+npm run build
+cd ..
 ```
 
-3. **Start the services**
+Set `DATABASE_URL`, `JWT_SECRET_KEY`, `FERNET_KEY`, and `OPENAI_API_KEY`. Use a random database password and match it to `POSTGRES_PASSWORD` if using the included PostgreSQL service. OpenAI fallback and encrypted model fields initialize even when their integrations are otherwise unused. Optional integration credentials can remain empty in validation. The current Tavily client initialization still needs `TAVILY_API_KEY` at app import time; making it optional requires an application change. Configure all OIDC fields together to enable browser login.
+
+For local backend development, start only PostgreSQL with `docker compose up -d postgres`, set `DATABASE_URL` to the host connection (`localhost:5432`), and use local writable checkpoint paths. Then:
 
 ```bash
-# Development (local PostgreSQL only)
-docker compose -f docker-compose.dev.yml up -d
-
-# Or full Docker deployment (app runs as raggr-blue; see docs/deployment.md)
-docker compose --profile blue up -d
+uv run --frozen aerich upgrade
+uv run --frozen python app.py
 ```
 
-4. **Access the application**
+See the [development guide](docs/development.md) for new database initialization, frontend development, and CI checks. For production configuration and blue/green services, see the [deployment guide](docs/deployment.md).
 
-Open `http://localhost:8080` in your browser.
+## Agent tools and integrations
 
-## Development
+Tools are selected per request by the tool router. The current tool families include:
 
-### Local Development Setup
+- Document retrieval (`simba_search`) and Tavily web search (`web_search`).
+- Obsidian note search/read/create, journal tasks, and saved user memory.
+- Mealie recipes, meal plans, and shopping lists.
+- Google Calendar and Gmail, using the OAuth token file; access is gated by the router.
+- SimbaKit/PetKit device status, weight, litter, and drinking history.
+- Beli restaurant lists; credentials are stored encrypted per profile.
+- Admin integrations: YNAB mirror SQL, Homebox inventory through `homebox-mcp`, and BookStack wiki through `bookstack-mcp`.
+
+The application also supports WhatsApp (Twilio), email (Mailgun), iMessage (SendBlue), scheduled messages, and Cronicle triggers. Integration availability and access depend on credentials and user role. Tool schemas being present does not imply an integration is configured.
+
+## Checks and operations
 
 ```bash
-# 1. Start PostgreSQL
-docker compose -f docker-compose.dev.yml up -d
-
-# 2. Set environment variables
-export DATABASE_URL="postgres://raggr:raggr_dev_password@localhost:5432/raggr"
-export CHROMADB_PATH="./chromadb"
-export $(grep -v '^#' .env | xargs)
-
-# 3. Install dependencies
-pip install -r requirements.txt
-cd raggr-frontend && yarn install && yarn build && cd ..
-
-# 4. Run migrations
-aerich upgrade
-
-# 5. Start the server
-python app.py
+make test             # backend unit tests; no deployment secrets required
+make evaluate         # deterministic context fixture evaluation
+make lint             # Ruff 0.8.2, matching pre-commit/CI
+make frontend-check   # locked install, renderer tests, TypeScript, build
+make validate-config  # production policy; prints setting names, not secrets
 ```
 
-See [docs/development.md](docs/development.md) for detailed development guide.
+Gitea Actions runs backend Ruff/unit/evaluation and frontend renderer/typecheck/build checks independently. Python dependencies come from `uv.lock`; frontend dependencies come from `raggr-frontend/package-lock.json` with `npm ci`.
 
-## Project Structure
+The production Makefile detects the active blue/green color for `make logs`, `make migrate`, and `make migrate-history`. `make redeploy` pulls the current branch and invokes `scripts/deploy.sh`; run it only when an actual deployment is intended. `make reindex` clears and rebuilds documents; `make reindex-obsidian` rebuilds only Obsidian documents. `make ynab-sync` performs a delta mirror sync and `make ynab-resync` a full download.
 
-```
-simbarag/
-├── app.py                 # Quart application entry point
-├── main.py                # RAG logic & LangChain agent
-├── llm.py                 # LLM client with Ollama/OpenAI
-├── aerich_config.py       # Database migration configuration
-│
-├── blueprints/            # API route blueprints
-│   ├── users/            # Authentication & authorization
-│   ├── conversation/     # Chat conversations
-│   └── rag/              # Document indexing
-│
-├── config/               # Configuration modules
-│   └── oidc_config.py   # OIDC authentication settings
-│
-├── utils/                # Reusable utilities
-│   ├── chunker.py       # Document chunking for embeddings
-│   ├── cleaner.py       # PDF cleaning and summarization
-│   ├── image_process.py # Image description with LLM
-│   └── request.py       # Paperless-NGX API client
-│
-├── scripts/              # Administrative scripts
-│   ├── add_user.py
-│   ├── user_message_stats.py
-│   ├── manage_vectorstore.py
-│   └── inspect_vector_store.py
-│
-├── raggr-frontend/       # React frontend
-│   └── src/
-│
-├── migrations/           # Database migrations
-│
-├── docs/                 # Documentation
-│   ├── index.md         # Documentation hub
-│   ├── development.md   # Development guide
-│   ├── deployment.md    # Deployment & migrations
-│   ├── VECTORSTORE.md   # Vector store management
-│   ├── MIGRATIONS.md    # Migration reference
-│   └── authentication.md # Authentication setup
-│
-├── docker-compose.yml        # Production compose
-├── docker-compose.dev.yml    # Development compose
-├── Dockerfile                # Production Dockerfile
-├── Dockerfile.dev            # Development Dockerfile
-├── CLAUDE.md                 # AI assistant instructions
-└── README.md                 # This file
-```
-
-## Key Technologies
-
-### Backend
-- **Quart** - Async Python web framework
-- **LangChain** - Agent framework with tool use
-- **Tortoise ORM** - Async ORM for PostgreSQL
-- **Aerich** - Database migration tool
-- **ChromaDB** - Vector database for embeddings
-- **OpenAI** - Embeddings & LLM (fallback)
-- **Ollama** - Local LLM (primary)
-
-### Frontend
-- **React 19** - UI framework
-- **Rsbuild** - Fast bundler
-- **Tailwind CSS** - Utility-first styling
-- **Axios** - HTTP client
-
-### Authentication
-- **Authelia** - OIDC provider
-- **LLDAP** - Lightweight LDAP server
-- **JWT** - Token-based auth
-
-## API Endpoints
-
-### Authentication
-- `GET /api/user/oidc/login` - Initiate OIDC login
-- `GET /api/user/oidc/callback` - OIDC callback handler
-- `POST /api/user/refresh` - Refresh JWT token
-
-### Conversations
-- `POST /api/conversation/` - Create conversation
-- `GET /api/conversation/` - List conversations
-- `GET /api/conversation/<id>` - Get conversation with messages
-- `POST /api/conversation/query` - Send message and get response
-
-### RAG (Admin Only)
-- `GET /api/rag/stats` - Vector store statistics
-- `POST /api/rag/index` - Index new documents
-- `POST /api/rag/reindex` - Clear and reindex all
-
-## Configuration
-
-### Environment Variables
-
-| Variable | Description | Default |
-|----------|-------------|---------|
-| `DATABASE_URL` | PostgreSQL connection string | `postgres://...` |
-| `CHROMADB_PATH` | ChromaDB storage path | `./chromadb` |
-| `OLLAMA_URL` | Ollama server URL | `http://localhost:11434` |
-| `OPENAI_API_KEY` | OpenAI API key | - |
-| `PAPERLESS_TOKEN` | Paperless-NGX API token | - |
-| `BASE_URL` | Paperless-NGX base URL | - |
-| `OIDC_ISSUER` | OIDC provider URL | - |
-| `OIDC_CLIENT_ID` | OIDC client ID | - |
-| `OIDC_CLIENT_SECRET` | OIDC client secret | - |
-| `JWT_SECRET_KEY` | JWT signing key | - |
-| `TAVILY_KEY` | Tavily web search API key | - |
-
-See `.env.example` for full list.
-
-## Scripts
-
-### User Management
-```bash
-# Add a new user
-python scripts/add_user.py
-
-# View message statistics
-python scripts/user_message_stats.py
-```
-
-### Vector Store Management
-```bash
-# Show vector store statistics
-python scripts/manage_vectorstore.py stats
-
-# Index new documents from Paperless
-python scripts/manage_vectorstore.py index
-
-# Clear and reindex everything
-python scripts/manage_vectorstore.py reindex
-
-# Inspect vector store contents
-python scripts/inspect_vector_store.py
-```
-
-See [docs/vectorstore.md](docs/vectorstore.md) for details.
-
-## Database Migrations
-
-```bash
-# Generate a new migration
-aerich migrate --name "describe_your_changes"
-
-# Apply pending migrations
-aerich upgrade
-
-# View migration history
-aerich history
-
-# Rollback last migration
-aerich downgrade
-```
-
-See [docs/deployment.md](docs/deployment.md) for detailed migration workflows.
-
-## LangChain Agent
-
-The conversational agent has access to two tools:
-
-1. **simba_search** - Query the vector store for Simba's documents
-   - Used for: Medical records, veterinary history, factual information
-
-2. **web_search** - Search the web via Tavily API
-   - Used for: Recent events, external knowledge, general questions
-
-The agent automatically selects the appropriate tool based on the user's query.
-
-## Authentication Flow
-
-```
-User → Authelia (OIDC) → Backend (JWT) → Frontend (localStorage)
-                ↓
-              LLDAP
-```
-
-1. User clicks "Login"
-2. Frontend redirects to Authelia
-3. User authenticates via Authelia (backed by LLDAP)
-4. Authelia redirects back with authorization code
-5. Backend exchanges code for OIDC tokens
-6. Backend issues JWT tokens
-7. Frontend stores tokens in localStorage
-
-## Contributing
-
-1. Fork the repository
-2. Create a feature branch
-3. Make your changes
-4. Run tests and linting
-5. Submit a pull request
-
-## Documentation
-
-- [Development Guide](docs/development.md) - Setup and development workflow
-- [Deployment Guide](docs/deployment.md) - Deployment and migrations
-- [Vector Store Guide](docs/vectorstore.md) - Managing the vector database
-- [Authentication Guide](docs/authentication.md) - OIDC and LDAP setup
-
-## License
-
-[Your License Here]
-
-## Acknowledgments
-
-- Built for Simba, the most important cat in the world 🐱
-- Powered by LangChain, ChromaDB, and the open-source community
+Further documentation: [pgvector management](docs/VECTORSTORE.md), [authentication](docs/authentication.md), [context management](docs/context-management.md), and [desktop client](docs/desktop-client.md).

app.py

@@ -17,6 +17,14 @@ from quart_jwt_extended import JWTManager, get_jwt_identity
 from tortoise import Tortoise
 
 import blueprints.artifacts
+import blueprints.integrations
+from blueprints.conversation.history import history_blueprint
+from blueprints.conversation.runs import (
+    drain_runs,
+    expire_runs,
+    runs_blueprint,
+)
+from blueprints.rag.source_api import sources_blueprint
 import blueprints.conversation
 import blueprints.conversation.logic
 import blueprints.email
@@ -75,6 +83,10 @@ app.register_blueprint(blueprints.scheduled_messages.scheduled_messages_blueprin
 app.register_blueprint(blueprints.cron.cron_blueprint)
 app.register_blueprint(blueprints.beli.beli_blueprint)
 app.register_blueprint(blueprints.artifacts.artifacts_blueprint)
+app.register_blueprint(blueprints.integrations.integrations_blueprint)
+app.register_blueprint(history_blueprint)
+app.register_blueprint(runs_blueprint)
+app.register_blueprint(sources_blueprint)
 
 
 async def _obsidian_sync_loop():
@@ -104,6 +116,7 @@ async def lifespan():
     logging.info("Initializing Tortoise ORM...")
     await Tortoise.init(config=TORTOISE_CONFIG)
     logging.info("Tortoise ORM initialized successfully")
+    await expire_runs()
 
     import aiosqlite
     from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
@@ -138,6 +151,9 @@ async def lifespan():
 
         yield
 
+        # Let in-flight chat runs finish while the agent and checkpointer are alive.
+        await drain_runs(app)
+
         for task in background_tasks:
             task.cancel()
 
@@ -254,4 +270,15 @@ async def get_messages():
 
 
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=8080, debug=True)
+    from hypercorn.asyncio import serve
+    from hypercorn.config import Config
+
+    config = Config()
+    config.bind = ["0.0.0.0:8080"]
+    config.accesslog = "-"
+    config.errorlog = "-"
+    # SSE subscribers reconnect to the new deployment, so connections close
+    # quickly; lifespan shutdown must outlast chat-run draining (drain_runs).
+    config.graceful_timeout = 10
+    config.shutdown_timeout = float(os.getenv("CHAT_RUN_DRAIN_SECONDS", "1800")) + 30
+    asyncio.run(serve(app, config))

blueprints/artifacts/models.py

@@ -8,13 +8,17 @@ class Artifact(Model):
     id = fields.UUIDField(primary_key=True)
     user = fields.ForeignKeyField("models.User", related_name="artifacts")
     conversation = fields.ForeignKeyField(
-        "models.Conversation", related_name="artifacts", on_delete=fields.CASCADE
+        "models.Conversation",
+        related_name="artifacts",
+        on_delete=fields.CASCADE,
+        db_index=True,
     )
     message = fields.ForeignKeyField(
         "models.ConversationMessage",
         related_name="artifacts",
         null=True,
         on_delete=fields.SET_NULL,
+        db_index=True,
     )
     tool = fields.CharField(max_length=255)
     kind = fields.CharField(max_length=64)

blueprints/conversation/__init__.py

@@ -7,8 +7,7 @@ import time
 import uuid
 from urllib.parse import quote, unquote
 
-from quart import Blueprint, current_app, jsonify, make_response, request
-from langchain_core.messages import convert_to_messages
+from quart import Blueprint, current_app, jsonify, request
 from quart_jwt_extended import get_jwt_identity
 
 import blueprints.users.models
@@ -33,13 +32,14 @@ from utils.s3_client import get_image as s3_get_image
 from utils.s3_client import upload_file as s3_upload_file
 from utils.s3_client import upload_image as s3_upload_image
 
+from .checkpoints import delete_thread_checkpoints
 from .logic import (
     add_message_to_conversation,
-    get_agent_for,
     persist_agent_tool_messages,
     tool_result_text,
 )
-from .memory import build_system_prompt_with_memories
+from . import runtime
+from .runtime import prepare_summary as _prepare_conversation_summary  # noqa: F401 — compatibility export
 from .models import (
     Conversation,
     PydConversation,
@@ -75,6 +75,7 @@ def _model_call_summary(call: dict) -> dict:
         "prefill_ms": round((first_chunk - start) * 1000) if first_chunk else None,
         "ms": round((end - start) * 1000) if end else None,
         "prompt_tokens": call.get("prompt_tokens"),
+        "cached_prompt_tokens": call.get("cached_prompt_tokens"),
         "output_tokens": call.get("output_tokens"),
         "model": call.get("model"),
         "provider": call.get("provider"),
@@ -134,23 +135,27 @@ conversation_blueprint = Blueprint(
     "conversation_api", __name__, url_prefix="/api/conversation"
 )
 
-# Detached persistence tasks. A strong reference keeps them from being garbage
-# collected mid-flight when the SSE stream is cancelled by a client disconnect.
-_persist_tasks: set[asyncio.Task] = set()
-
-
-def _persist_in_background(coro) -> asyncio.Task:
-    task = asyncio.create_task(coro)
-    _persist_tasks.add(task)
-    task.add_done_callback(_persist_tasks.discard)
-    return task
-
 
 async def _get_owned_conversation(conversation_id, user_id) -> Conversation | None:
     """Fetch a conversation only if it belongs to the given user."""
+    try:
+        conversation_id = uuid.UUID(str(conversation_id))
+    except ValueError:
+        return None
     return await Conversation.get_or_none(id=conversation_id, user_id=user_id)
 
 
+async def _owns_storage_key(key: str, user_id) -> bool:
+    """True if `key` is stored under a conversation the user owns.
+
+    Every uploaded object lives under conversations/<conversation id>/.
+    """
+    parts = key.split("/")
+    if len(parts) < 3 or parts[0] != "conversations" or ".." in parts:
+        return False
+    return await _get_owned_conversation(parts[1], user_id) is not None
+
+
 def _build_user_message(
     query_text: str,
     image_block: dict | None = None,
@@ -170,38 +175,30 @@ def _build_user_message(
     return {"role": "user", "content": content}
 
 
-async def _prepare_conversation_summary(agent, payload, config) -> None:
-    """Archive completed checkpoint turns before adding this request's message."""
-    store = getattr(current_app, "conversation_summary_store", None)
-    if store is None:
-        return
-    configurable = config["configurable"]
-    try:
-        checkpoint = await agent.aget_state(config)
-        history = convert_to_messages(checkpoint.values.get("messages", []))
-        state = await store.update(
-            configurable["user_id"],
-            configurable["thread_id"],
-            history + convert_to_messages(payload["messages"]),
-        )
-        configurable["conversation_summary"] = state.as_message()
-    except Exception:
-        # The original checkpoint still supplies recent history if the optional
-        # summary store is unavailable. Don't fail an otherwise valid request.
-        logging.exception("Failed to prepare conversation summary")
-
-
 @conversation_blueprint.post("/query")
 @jwt_access_required
 async def query():
     current_user_uuid = get_jwt_identity()
     user = await blueprints.users.models.User.get(id=current_user_uuid)
-    data = await request.get_json()
+    data = await request.get_json(silent=True)
+    if not isinstance(data, dict):
+        return jsonify({"error": "JSON object required"}), 400
     query = data.get("query")
+    if not isinstance(query, str) or not query.strip():
+        return jsonify({"error": "query must be a non-empty string"}), 400
     conversation_id = data.get("conversation_id")
+    if not isinstance(conversation_id, str):
+        return jsonify({"error": "conversation not found"}), 404
     conversation = await _get_owned_conversation(conversation_id, current_user_uuid)
     if conversation is None:
         return jsonify({"error": "conversation not found"}), 404
+
+    from .run_models import ChatRun
+
+    # A durable run owns this checkpoint thread (possibly in another process).
+    if await ChatRun.filter(active_key=str(conversation.id)).exists():
+        return jsonify({"error": "A run is already active in this conversation"}), 409
+
     await add_message_to_conversation(
         conversation=conversation,
         message=query,
@@ -210,22 +207,9 @@ async def query():
     )
 
     with time_response(stream=False):
-        system_prompt = await build_system_prompt_with_memories(
-            str(user.id), query=query or ""
-        )
-        payload = {"messages": [_build_user_message(query)]}
-        agent_config = {
-            "configurable": {
-                "user_id": str(user.id),
-                "is_admin": user.is_admin(),
-                "thread_id": str(conversation.id),
-                "system_prompt": system_prompt,
-            }
-        }
-
-        main_agent = get_agent_for(user)
-        await _prepare_conversation_summary(main_agent, payload, agent_config)
-        response = await main_agent.ainvoke(payload, config=agent_config)
+        async with runtime.thread_lock(str(conversation.id)):
+            turn = await runtime.prepare_agent_turn(user, query, str(conversation.id))
+            response = await runtime.invoke_prepared(turn)
         message = extract_agent_text(response.get("messages", [])[-1].content)
         await persist_agent_tool_messages(
             conversation, user, response.get("messages", [])
@@ -252,12 +236,16 @@ async def upload_image():
 
     if not file or not conversation_id:
         return jsonify({"error": "file and conversation_id are required"}), 400
+    if await _get_owned_conversation(conversation_id, current_user_uuid) is None:
+        return jsonify({"error": "conversation not found"}), 404
 
     file_bytes = file.read()
     content_type = file.content_type or "image/jpeg"
 
     try:
-        processed_bytes, output_content_type = process_image(file_bytes, content_type)
+        processed_bytes, output_content_type = await asyncio.to_thread(
+            process_image, file_bytes, content_type
+        )
     except ImageValidationError as e:
         return jsonify({"error": str(e)}), 400
 
@@ -311,8 +299,8 @@ async def upload_attachment():
 
     if content_type in IMAGE_TYPES:
         try:
-            processed_bytes, output_content_type = process_image(
-                file_bytes, content_type
+            processed_bytes, output_content_type = await asyncio.to_thread(
+                process_image, file_bytes, content_type
             )
         except Exception as e:
             return jsonify({"error": f"Could not process image: {e}"}), 400
@@ -355,6 +343,8 @@ async def upload_attachment():
 @conversation_blueprint.get("/image/<path:image_key>")
 @jwt_access_required
 async def serve_image(image_key: str):
+    if not await _owns_storage_key(image_key, get_jwt_identity()):
+        return jsonify({"error": "image not found"}), 404
     url = await s3_presigned_url(image_key)
     return jsonify({"url": url})
 
@@ -362,23 +352,31 @@ async def serve_image(image_key: str):
 @conversation_blueprint.post("/stream-query")
 @jwt_access_required
 async def stream_query():
-    current_user_uuid = get_jwt_identity()
-    user = await blueprints.users.models.User.get(id=current_user_uuid)
-    data = await request.get_json()
+    # Compatibility adapter: disconnect detaches; explicit cancellation uses /runs.
+    from .runs import create_run, replay_response, RunConflict
+
+    try:
+        run, _ = await create_run(
+            current_app._get_current_object(),
+            get_jwt_identity(),
+            await request.get_json(),
+        )
+    except RunConflict as exc:
+        return jsonify({"error": str(exc)}), exc.status
+    return await replay_response(run, 0)
+
+
+async def _chat_run_stream(data, user, conversation, save_user, save_answer):
     query_text = data.get("query")
-    conversation_id = data.get("conversation_id")
     image_key = data.get("image_key")
     file_key = data.get("file_key")
-    conversation = await _get_owned_conversation(conversation_id, current_user_uuid)
-    if conversation is None:
-        return jsonify({"error": "conversation not found"}), 404
 
     # An attached file is extracted and appended to the agent's user turn; the
     # saved message only records its name.
     saved_text = query_text or ""
     if file_key:
         if not _file_key_belongs_to(file_key, conversation.id):
-            return jsonify({"error": "invalid file_key"}), 400
+            raise ValueError("invalid file_key")
         try:
             file_name, file_text = await _load_attached_file(file_key)
             block = attachment_block(file_name, file_text)
@@ -389,41 +387,24 @@ async def stream_query():
         query_text = f"{query_text or 'The user attached this file.'}\n\n{block}"
         saved_text = f"{saved_text}\n\n📎 {file_name}".strip()
 
-    await add_message_to_conversation(
-        conversation=conversation,
-        message=saved_text,
-        speaker="user",
-        user=user,
-        image_key=image_key,
-    )
+    await save_user(saved_text, image_key)
 
     # If an image was uploaded, attach it to the agent request directly
     image_block = None
     if image_key:
         try:
             image_bytes, _ = await s3_get_image(image_key)
-            image_block = image_content_block(image_bytes)
+            image_block = await asyncio.to_thread(image_content_block, image_bytes)
         except Exception as e:
             logging.error(f"Failed to load image {image_key}: {e}")
             query_text = f"{query_text or ''}\n\n[An image was sent but could not be loaded.]".strip()
 
-    system_prompt = await build_system_prompt_with_memories(
-        str(user.id), query=saved_text or ""
+    turn = await runtime.prepare_agent_turn(
+        user,
+        saved_text or "",
+        str(conversation.id),
+        messages=[_build_user_message(query_text or "", image_block)],
     )
-    payload = {"messages": [_build_user_message(query_text or "", image_block)]}
-    agent_config = {
-        "configurable": {
-            "user_id": str(user.id),
-            "is_admin": user.is_admin(),
-            "thread_id": str(conversation.id),
-            "system_prompt": system_prompt,
-        }
-    }
-    # Capture the agent reference while the request/app context is still active.
-    # Quart exits the app context before iterating the SSE response body, so
-    # current_app cannot be accessed inside event_generator().
-    main_agent = get_agent_for(user)
-    await _prepare_conversation_summary(main_agent, payload, agent_config)
 
     async def event_generator():
         final_message = None
@@ -447,15 +428,16 @@ async def stream_query():
         content_chunks = 0
         model_name: str | None = None
         cancelled = False
-        persistence_task: asyncio.Task | None = None
+        persist_attempted = False
 
         async def persist(text: str) -> None:
-            await add_message_to_conversation(
-                conversation=conversation,
-                message=text,
-                speaker="simba",
-                user=user,
-            )
+            # At most one attempt: a save cancelled mid-flight must not be retried
+            # by the cancellation handler, or the answer could be stored twice.
+            nonlocal persist_attempted
+            if persist_attempted:
+                return
+            persist_attempted = True
+            await save_answer(text)
 
         def _log_summary(*, error: str | None = None) -> None:
             now = time.monotonic()
@@ -499,9 +481,7 @@ async def stream_query():
             )
 
         try:
-            aiter = main_agent.astream_events(
-                payload, version="v2", config=agent_config
-            ).__aiter__()
+            aiter = runtime.stream_prepared(turn).__aiter__()
             while True:
                 # Only a silent model call counts as a stall. A model that is
                 # thinking (or streaming empty chunks) keeps resetting the clock,
@@ -600,6 +580,10 @@ async def stream_query():
                         if usage:
                             call["prompt_tokens"] = usage.get("input_tokens")
                             call["output_tokens"] = usage.get("output_tokens")
+                            # Prompt-cache hits, when the provider reports them.
+                            call["cached_prompt_tokens"] = (
+                                usage.get("input_token_details") or {}
+                            ).get("cache_read")
                             call["reasoning_tokens"] = (
                                 usage.get("output_token_details") or {}
                             ).get("reasoning")
@@ -645,9 +629,8 @@ async def stream_query():
 
             final_message = final_message or streamed
             if final_message:
-                # Shield the single persistence task across client disconnects.
-                persistence_task = _persist_in_background(persist(final_message))
-                await asyncio.shield(persistence_task)
+                # The durable worker owns persistence independently of subscribers.
+                await persist(final_message)
                 yield f"data: {json.dumps({'type': 'response', 'message': final_message})}\n\n"
             else:
                 yield f"data: {json.dumps({'type': 'error', 'message': 'No response generated'})}\n\n"
@@ -657,9 +640,9 @@ async def stream_query():
             count_result(stream=True, result="cancelled")
             observe_response(stream=True, seconds=time.monotonic() - t0)
             _log_summary()
-            text = final_message or streamed
-            if text and persistence_task is None:
-                _persist_in_background(persist(text))
+            text = final_message or (streamed + (tag_buf if not in_thinking else ""))
+            if text:
+                await persist(text)
             raise
         except Exception as e:
             count_result(stream=True, result="error")
@@ -673,19 +656,8 @@ async def stream_query():
         _log_summary(error=None if final_message else "No response generated")
         yield "data: [DONE]\n\n"
 
-    response = await make_response(
-        event_generator(),
-        200,
-        {
-            "Content-Type": "text/event-stream",
-            "Cache-Control": "no-cache",
-            "X-Accel-Buffering": "no",
-        },
-    )
-    # Quart's default 60s response deadline cancels even active SSE streams.
-    # Silent model calls are bounded separately by MODEL_STALL_TIMEOUT.
-    response.timeout = None
-    return response
+    async for frame in event_generator():
+        yield frame
 
 
 @conversation_blueprint.route("/<conversation_id>")
@@ -806,5 +778,11 @@ async def delete_conversation(conversation_id: str):
     store = getattr(current_app, "conversation_summary_store", None)
     if store is not None:
         await store.delete(str(user_uuid), str(conversation.id))
+    thread_id = str(conversation.id)
     await conversation.delete()
+    checkpointer = getattr(
+        getattr(current_app, "main_agent", None), "checkpointer", None
+    )
+    if checkpointer is not None:
+        await delete_thread_checkpoints(checkpointer, thread_id)
     return jsonify({"success": True})

blueprints/conversation/agents.py

@@ -23,13 +23,14 @@ from blueprints.conversation.artifacts_util import (
     note_title,
     obsidian_source_ref,
     search_documents_artifact,
+    search_results_model_context,
     truncate,
     unique,
 )
 from blueprints.conversation.context import RESPONSE_RESERVE_TOKENS, build_context
 from blueprints.conversation.memory import save_memory
 from blueprints.conversation.prompts import SIMBA_SYSTEM_PROMPT
-from blueprints.conversation.tool_routing import ToolRouter
+from blueprints.conversation.tool_routing import TOOL_ROUTING_MODES, ToolRouter
 from blueprints.rag.logic import query_vector_store
 from blueprints.ynab.sync import ynab_configured
 from blueprints.ynab.tools import YNAB_TOOLS
@@ -140,18 +141,32 @@ async def web_search(query: str) -> str:
 
 
 @tool(response_format="content_and_artifact")
-async def simba_search(query: str):
+async def simba_search(
+    query: str, start_date: str = "", end_date: str = "", k: int = 8
+):
     """Search Simba's medical records, vet documents, and household documents.
 
     Args:
         query: Question or information need
+        start_date: Inclusive document date YYYY-MM-DD, or empty for all dates.
+        end_date: Inclusive document date YYYY-MM-DD, or empty for all dates.
+        k: Maximum results, from 1 to 20.
     """
-    print(f"[SIMBA SEARCH] Tool called with query: {query}")
-    serialized, docs = await query_vector_store(query=query, source="paperless")
-    print(f"[SIMBA SEARCH] Found {len(docs)} documents")
-    print(f"[SIMBA SEARCH] Serialized result length: {len(serialized)}")
-    print(f"[SIMBA SEARCH] First 200 chars: {serialized[:200]}")
-    return serialized, search_documents_artifact(query, "paperless", docs)
+    try:
+        kwargs = (
+            {"start_date": start_date, "end_date": end_date, "k": k}
+            if start_date or end_date or k != 8
+            else {}
+        )
+        serialized, docs = await query_vector_store(
+            query=query, source="paperless", **kwargs
+        )
+    except ValueError as exc:
+        return str(exc), None
+    artifact = search_documents_artifact(query, "paperless", docs)
+    if artifact is not None:
+        artifact["model_context"] = search_results_model_context(docs, "paperless")
+    return serialized, artifact
 
 
 MEALIE_NOT_CONFIGURED = "Mealie integration is not configured. Please set MEALIE_BASE_URL and MEALIE_API_TOKEN environment variables."
@@ -453,19 +468,34 @@ def _note_artifact(path: str, parsed: dict, title: str | None = None) -> dict:
 
 
 @tool(response_format="content_and_artifact")
-async def obsidian_search_notes(query: str):
+async def obsidian_search_notes(
+    query: str, start_date: str = "", end_date: str = "", k: int = 8
+):
     """Search personal Obsidian vault notes.
 
     Args:
         query: Search query
+        start_date: Inclusive note date YYYY-MM-DD, or empty for all dates.
+        end_date: Inclusive note date YYYY-MM-DD, or empty for all dates.
+        k: Maximum results, from 1 to 20.
     """
     if not obsidian_enabled:
         return OBSIDIAN_NOT_CONFIGURED, None
 
     try:
         # Query vector store filtered to obsidian source only
-        serialized, docs = await query_vector_store(query=query, source="obsidian")
-        return serialized, search_documents_artifact(query, "obsidian", docs)
+        kwargs = (
+            {"start_date": start_date, "end_date": end_date, "k": k}
+            if start_date or end_date or k != 8
+            else {}
+        )
+        serialized, docs = await query_vector_store(
+            query=query, source="obsidian", **kwargs
+        )
+        artifact = search_documents_artifact(query, "obsidian", docs)
+        if artifact is not None:
+            artifact["model_context"] = search_results_model_context(docs, "obsidian")
+        return serialized, artifact
 
     except Exception as e:
         return f"Error searching Obsidian notes: {str(e)}", None
@@ -1160,16 +1190,24 @@ def build_agent(checkpointer=None, extra_tools=()):
     """
     extra_tools = list(extra_tools)
     # Keep the guarded Google tools executable for admins even when no MCP tools
-    # are configured and app.admin_agent aliases app.main_agent. The router never
-    # binds their schemas for a regular user. Extra tools exist only on the admin
-    # graph and are independently gated by the same invocation config.
+    # are configured and app.admin_agent aliases app.main_agent. Neither routing
+    # mode binds their schemas for a regular user. Extra tools exist only on the
+    # admin graph and are independently gated by the same invocation config.
     agent_tools = tools + extra_tools
     router = ToolRouter(agent_tools, admin_tools=extra_tools)
+    mode = os.getenv("TOOL_ROUTING_MODE", "static").strip().lower()
+    if mode not in TOOL_ROUTING_MODES:
+        raise ValueError(
+            f"TOOL_ROUTING_MODE must be one of {TOOL_ROUTING_MODES}, got {mode!r}"
+        )
+    dynamic = mode == "dynamic"
 
     def routed_model(messages, config: RunnableConfig):
-        selected = router.select(
-            messages,
-            is_admin=(config or {}).get("configurable", {}).get("is_admin", False),
+        is_admin = (config or {}).get("configurable", {}).get("is_admin", False)
+        selected = (
+            router.select(messages, is_admin=is_admin)
+            if dynamic
+            else router.select_static(is_admin=is_admin)
         )
         schemas = [convert_to_openai_tool(t) for t in selected]
         fallback = openai_fallback.bind_tools(selected)
@@ -1188,6 +1226,6 @@ def build_agent(checkpointer=None, extra_tools=()):
 
     return create_react_agent(
         model=select_model,
-        tools=agent_tools + [router.discovery],
+        tools=agent_tools + ([router.discovery] if dynamic else []),
         checkpointer=checkpointer,
     )

blueprints/conversation/artifacts_util.py

@@ -143,36 +143,64 @@ def vault_relative_path(filepath: str) -> str:
         return filepath
 
 
+def _result_identity(doc, source: str) -> dict:
+    """Identifying fields shared by the UI snippet and the model's full-text view."""
+    from blueprints.rag.retrieval import citation_url
+
+    meta = doc.metadata or {}
+    result = {}
+    if url := citation_url(doc):
+        result["url"] = url
+        result["source_id"] = str(doc.id)
+    if headings := [meta[k] for k in ("h1", "h2", "h3") if meta.get(k)]:
+        result["headings"] = headings
+    if source == "obsidian":
+        filepath = meta.get("filepath") or ""
+        path = vault_relative_path(filepath) if filepath else ""
+        result["title"] = note_title(path, meta) if path else "Untitled note"
+        if path:
+            result["path"] = path
+        created = meta.get("created_at")
+        if isinstance(created, str) and created:
+            result["date"] = created[:10]
+    else:
+        filename = meta.get("filename") or ""
+        result["title"] = Path(filename).stem or filename or "Document"
+        if filename:
+            result["filename"] = filename
+        if meta.get("document_type"):
+            result["document_type"] = str(meta["document_type"])
+        created = epoch_to_date(meta.get("created_date"))
+        if created:
+            result["date"] = created
+    return result
+
+
 def search_documents_artifact(query: str, source: str, docs) -> dict | None:
     """Build a `search.documents` artifact from vector-store Documents."""
     if not docs:
         return None
-    results = []
-    for doc in docs:
-        meta = doc.metadata or {}
-        result = {"snippet": snippet(doc.page_content)}
-        if source == "obsidian":
-            filepath = meta.get("filepath") or ""
-            path = vault_relative_path(filepath) if filepath else ""
-            result["title"] = note_title(path, meta) if path else "Untitled note"
-            if path:
-                result["path"] = path
-            created = meta.get("created_at")
-            if isinstance(created, str) and created:
-                result["date"] = created[:10]
-        else:
-            filename = meta.get("filename") or ""
-            result["title"] = Path(filename).stem or filename or "Document"
-            if filename:
-                result["filename"] = filename
-            if meta.get("document_type"):
-                result["document_type"] = str(meta["document_type"])
-            created = epoch_to_date(meta.get("created_date"))
-            if created:
-                result["date"] = created
-        results.append(result)
+    results = [
+        {"snippet": snippet(doc.page_content), **_result_identity(doc, source)}
+        for doc in docs
+    ]
     return make_artifact(
         "search.documents",
         f"Search: {query}",
         {"query": query, "source": source, "results": results},
     )
+
+
+def search_results_model_context(docs, source: str) -> dict:
+    """Full chunk text for the model, keyed like `search_documents_artifact`'s results.
+
+    Set as a tool's `artifact["model_context"]` so that under compaction (see
+    compact_tool_result in blueprints/conversation/context.py) the model still
+    sees the full chunk text it retrieved, not the UI's 400-char snippets.
+    """
+    return {
+        "results": [
+            {**_result_identity(doc, source), "content": doc.page_content}
+            for doc in docs
+        ]
+    }

blueprints/conversation/checkpoints.py

@@ -0,0 +1,87 @@
+"""Keeps the LangGraph checkpoint DB from growing without bound.
+
+Every `AsyncSqliteSaver` row stores a full snapshot of the graph's channel
+values (the complete message list so far), not a delta — so an unpruned
+thread's checkpoints grow roughly linearly with turn count.
+"""
+
+import logging
+import os
+
+DEFAULT_KEEP_PER_THREAD = 20
+
+
+def keep_per_thread() -> int:
+    raw = os.getenv("CHECKPOINT_KEEP_PER_THREAD", str(DEFAULT_KEEP_PER_THREAD))
+    try:
+        value = int(raw)
+    except ValueError:
+        value = -1
+    if value < 1:
+        logging.warning(
+            "CHECKPOINT_KEEP_PER_THREAD=%r is invalid; falling back to %d",
+            raw,
+            DEFAULT_KEEP_PER_THREAD,
+        )
+        return DEFAULT_KEEP_PER_THREAD
+    return value
+
+
+async def prune_thread_checkpoints(
+    checkpointer, thread_id: str, keep: int | None = None
+) -> None:
+    """Keep only the newest `keep` checkpoints (and their writes) for a thread.
+
+    Safe to call after every turn: each checkpoint row is a self-contained
+    channel snapshot rather than a delta, and `aget_tuple`/`aget_state` only
+    ever read the single newest row for a (thread_id, checkpoint_ns) pair, so
+    dropping older rows never affects state reads. Reuses the saver's own
+    lock so this never races its aput/aget_tuple calls.
+    """
+    keep = keep_per_thread() if keep is None else keep
+    thread_id = str(thread_id)
+    conn = checkpointer.conn
+    async with checkpointer.lock, conn.cursor() as cur:
+        await cur.execute(
+            "SELECT DISTINCT checkpoint_ns FROM checkpoints WHERE thread_id = ?",
+            (thread_id,),
+        )
+        namespaces = [row[0] for row in await cur.fetchall()]
+        for ns in namespaces:
+            await cur.execute(
+                """
+                DELETE FROM checkpoints
+                WHERE thread_id = ? AND checkpoint_ns = ?
+                AND checkpoint_id NOT IN (
+                    SELECT checkpoint_id FROM checkpoints
+                    WHERE thread_id = ? AND checkpoint_ns = ?
+                    ORDER BY checkpoint_id DESC
+                    LIMIT ?
+                )
+                """,
+                (thread_id, ns, thread_id, ns, keep),
+            )
+            await cur.execute(
+                """
+                DELETE FROM writes
+                WHERE thread_id = ? AND checkpoint_ns = ?
+                AND checkpoint_id NOT IN (
+                    SELECT checkpoint_id FROM checkpoints
+                    WHERE thread_id = ? AND checkpoint_ns = ?
+                )
+                """,
+                (thread_id, ns, thread_id, ns),
+            )
+        await conn.commit()
+
+
+async def delete_thread_checkpoints(checkpointer, thread_id: str) -> None:
+    """Delete a conversation's checkpoint thread; never raises.
+
+    Called from conversation delete, which must still succeed even if the
+    checkpoint store is unreachable or the thread is already gone.
+    """
+    try:
+        await checkpointer.adelete_thread(str(thread_id))
+    except Exception:
+        logging.exception("Failed to delete checkpoint thread %s", thread_id)

blueprints/conversation/context.py

@@ -29,6 +29,10 @@ EARLIER_MESSAGE_MAX_CHARS = 4000
 TOOL_RESULT_MAX_CHARS = 12000
 TRUNCATION_NOTE = "\n[truncated: earlier context]"
 IMAGE_PLACEHOLDER = "[image from an earlier message]"
+# On overflow, drop old turns down to about this share of the history budget.
+TRIM_LOW_WATERMARK = 0.65
+# Upper bound on the summary ledger message (summary.STATE_TOKEN_LIMIT + wrapper).
+SUMMARY_RESERVE_TOKENS = 2000
 
 
 def estimate_tokens(value) -> int:
@@ -340,46 +344,48 @@ def _summary(turns: list[list[BaseMessage]], limit: int = 2400) -> AIMessage | N
     )
 
 
-def trim_history(
-    messages: list[BaseMessage],
-    *,
-    token_budget: int = 12000,
-    summary_message: AIMessage | None = None,
-) -> list[BaseMessage]:
-    """Keep the current turn, then recent whole turns, within an estimated budget.
+def _stable_cut(costs: list[int], room: int, step: int) -> int:
+    """Return how many of the oldest prior turns to drop so the rest fit ``room``.
 
-    The current user message and tool call arguments are never truncated. Fail
-    explicitly if these mandatory inputs cannot fit, rather than lose the task.
+    Cut points sit where the cumulative cost from the oldest turn crosses a
+    multiple of ``step``. They depend only on turns already in history, so the
+    cut stays put as new turns arrive and only moves, by about ``step`` tokens,
+    once the retained turns overflow again. That keeps the prompt prefix
+    byte-identical across several turns for llama-server's prompt cache.
     """
-    if summary_message is not None:
-        if not isinstance(summary_message, AIMessage) or summary_message.tool_calls:
-            raise ValueError(
-                "Conversation summary must be assistant-role evidence without tool calls"
-            )
-        summary_cost = message_tokens(summary_message)
-        if summary_cost > token_budget:
-            logging.info("Omitting optional conversation summary: insufficient budget")
-            return trim_history(messages, token_budget=token_budget)
-        try:
-            trimmed = trim_history(messages, token_budget=token_budget - summary_cost)
-        except ValueError:
-            # Historical evidence must never evict a current request that fits
-            # without it. Required current-turn metadata retains priority.
-            logging.info(
-                "Omitting optional conversation summary to preserve current request"
-            )
-            return trim_history(messages, token_budget=token_budget)
-        if (
-            trimmed
-            and isinstance(trimmed[0], AIMessage)
-            and isinstance(trimmed[0].content, str)
-            and trimmed[0].content.startswith("Earlier conversation excerpts")
-        ):
-            trimmed = trimmed[1:]
-        return [summary_message, *trimmed]
+    remaining = sum(costs)
+    if remaining <= room:
+        return 0
+    total = 0
+    for i, turn_cost in enumerate(costs):
+        before = total
+        total += turn_cost
+        remaining -= turn_cost
+        if total // step > before // step and remaining <= room:
+            return i + 1
+    # No cut point leaves room for any prior turn (e.g. a heavy current turn).
+    # Fall back to keeping as many recent whole turns as fit.
+    remaining = sum(costs)
+    for i, turn_cost in enumerate(costs):
+        if remaining <= room:
+            return i
+        remaining -= turn_cost
+    return len(costs)
+
+
+def _trim(
+    messages: list[BaseMessage], token_budget: int, reserve: int = 0
+) -> tuple[list[BaseMessage], int]:
+    """Trim to ``token_budget - reserve``; return the messages and dropped turns.
+
+    ``reserve`` shrinks the space without moving the hysteresis cut points, so a
+    caller-supplied summary of varying size doesn't shift which turns are kept.
+    """
+    step = max(1, int(token_budget * (1 - TRIM_LOW_WATERMARK)))
+    token_budget -= reserve
     turns = [_complete_exchanges(turn) for turn in _turns(messages)]
     if not turns:
-        return []
+        return [], 0
     original_current = turns[-1]
     current = original_current
     current = [
@@ -421,19 +427,21 @@ def trim_history(
             else:
                 high = mid - 1
         current = compact(low)
-    selected = []
     used = cost(current)
-    prior = turns[:-1]
-    # Leave a small share for excerpts of older turns only when necessary.
-    summary_reserve = min(1000, max(0, token_budget - used))
-    for turn in reversed(prior):
-        shrunk = [_shrink(m, EARLIER_MESSAGE_MAX_CHARS, earlier=True) for m in turn]
-        turn_cost = cost(shrunk)
-        if used + turn_cost > token_budget - summary_reserve:
-            break
-        selected.insert(0, shrunk)
-        used += turn_cost
-    dropped = prior[: len(prior) - len(selected)]
+    prior = [
+        [_shrink(m, EARLIER_MESSAGE_MAX_CHARS, earlier=True) for m in turn]
+        for turn in turns[:-1]
+    ]
+    costs = [cost(turn) for turn in prior]
+    if used + sum(costs) <= token_budget:
+        cut = 0
+    else:
+        # Leave a small share for excerpts of older turns only when dropping.
+        summary_reserve = min(1000, max(0, token_budget - used))
+        cut = _stable_cut(costs, token_budget - summary_reserve - used, step)
+    selected = prior[cut:]
+    used += sum(costs[cut:])
+    dropped = turns[:cut]
     summary = _summary(dropped)
     result = [m for turn in selected for m in turn] + current
     if summary and used + message_tokens(summary) <= token_budget:
@@ -451,7 +459,68 @@ def trim_history(
             }
         )
     )
-    return result
+    return result, len(dropped)
+
+
+def trim_history(
+    messages: list[BaseMessage],
+    *,
+    token_budget: int = 12000,
+    summary_message: AIMessage | None = None,
+) -> list[BaseMessage]:
+    """Keep the current turn, then recent whole turns, within an estimated budget.
+
+    The current user message and tool call arguments are never truncated. Fail
+    explicitly if these mandatory inputs cannot fit, rather than lose the task.
+    When the budget overflows, old turns are dropped down to a lower watermark
+    so the retained prefix stays stable (and cacheable) for several turns.
+    """
+    if summary_message is not None:
+        if not isinstance(summary_message, AIMessage) or summary_message.tool_calls:
+            raise ValueError(
+                "Conversation summary must be assistant-role evidence without tool calls"
+            )
+    trimmed, dropped = _trim(messages, token_budget)
+    if summary_message is None or not dropped:
+        # The summary only duplicates retained turns when nothing was dropped,
+        # and it changes every turn, which would defeat prompt caching.
+        return trimmed
+    summary_cost = message_tokens(summary_message)
+    if summary_cost > token_budget:
+        logging.info("Omitting optional conversation summary: insufficient budget")
+        return trimmed
+    # Reserve a fixed ceiling, not the ledger's current size, so a shrinking
+    # ledger can't pull the cut back and change the retained prefix.
+    for reserve in dict.fromkeys(
+        (max(summary_cost, SUMMARY_RESERVE_TOKENS), summary_cost)
+    ):
+        try:
+            with_room, _ = _trim(messages, token_budget, reserve=reserve)
+            break
+        except ValueError:
+            continue
+    else:
+        # Historical evidence must never evict a current request that fits
+        # without it. Required current-turn metadata retains priority.
+        logging.info(
+            "Omitting optional conversation summary to preserve current request"
+        )
+        return trimmed
+    # The ledger supersedes excerpts of dropped turns; whenever it is included
+    # they are consistently absent, so this doesn't disturb the cached prefix.
+    if (
+        with_room
+        and isinstance(with_room[0], AIMessage)
+        and isinstance(with_room[0].content, str)
+        and with_room[0].content.startswith("Earlier conversation excerpts")
+    ):
+        with_room = with_room[1:]
+    # The ledger changes every turn, so place it just before the current turn:
+    # system, tools and retained history then stay byte-identical for the
+    # prompt cache. Turns start at a HumanMessage, so the last one opens the
+    # current turn and no tool exchange is split.
+    start = max(i for i, m in enumerate(with_room) if isinstance(m, HumanMessage))
+    return [*with_room[:start], summary_message, *with_room[start:]]
 
 
 def build_context(

blueprints/conversation/history.py

@@ -0,0 +1,277 @@
+"""Owner-scoped history and memory management; register history_blueprint in app."""
+
+import re
+from uuid import UUID
+
+from quart import Blueprint, Response, jsonify, request
+from quart_jwt_extended import get_jwt_identity
+from tortoise.expressions import Q
+
+from blueprints.users.decorators import jwt_access_required
+from .models import Conversation, ConversationMessage, Speaker, UserMemory
+
+history_blueprint = Blueprint("history_api", __name__, url_prefix="/api/history")
+
+
+def valid_id(value):
+    try:
+        return UUID(str(value))
+    except (ValueError, TypeError, AttributeError):
+        return None
+
+
+def pagination(limit=20, offset=0):
+    try:
+        limit, offset = int(limit), int(offset)
+    except (TypeError, ValueError):
+        raise ValueError("limit and offset must be integers") from None
+    if not 1 <= limit <= 100 or offset < 0:
+        raise ValueError("limit must be 1–100 and offset must be nonnegative")
+    return limit, offset
+
+
+def display_text(message):
+    return (
+        message.text.split("\n", 1)[0]
+        if message.speaker == Speaker.TOOL
+        else message.text
+    )
+
+
+def snippet(text, search):
+    index = text.casefold().find(search.casefold())
+    start = max(0, index - 60)
+    end = min(len(text), start + 200)
+    return ("…" if start else "") + text[start:end] + ("…" if end < len(text) else "")
+
+
+async def list_owned_conversations(user_id, search=None, limit=20, offset=0):
+    """Stable pinned-first pagination. DISTINCT runs before LIMIT on message joins."""
+    limit, offset = pagination(limit, offset)
+    if search is not None and (not isinstance(search, str) or len(search) > 500):
+        raise ValueError("search must be text of at most 500 characters")
+    search = (search or "").strip()
+    query = Conversation.filter(user_id=user_id)
+    if search:
+        query = query.filter(
+            Q(name__icontains=search) | Q(messages__text__icontains=search)
+        )
+    conversations = (
+        await query.distinct()
+        .order_by("-pinned", "-updated_at", "-id")
+        .offset(offset)
+        .limit(limit)
+    )
+    rows = []
+    for conversation in conversations:
+        row = {
+            "id": str(conversation.id),
+            "name": conversation.name,
+            "pinned": conversation.pinned,
+            "created_at": conversation.created_at.isoformat(),
+            "updated_at": conversation.updated_at.isoformat(),
+        }
+        if search:
+            match = (
+                await ConversationMessage.filter(
+                    conversation_id=conversation.id, text__icontains=search
+                )
+                .order_by("created_at", "id")
+                .first()
+            )
+            row["snippet"] = (
+                snippet(display_text(match), search)
+                if match
+                else snippet(conversation.name, search)
+            )
+            if (
+                match
+                and match.speaker == Speaker.TOOL
+                and search.casefold() not in display_text(match).casefold()
+            ):
+                row["snippet"] = (
+                    "Match in stored tool context (hidden in chat): "
+                    + display_text(match)[:150]
+                )
+            row["match_message_id"] = str(match.id) if match else None
+        rows.append(row)
+    return rows
+
+
+@history_blueprint.get("/conversations")
+@jwt_access_required
+async def conversations():
+    try:
+        rows = await list_owned_conversations(
+            get_jwt_identity(),
+            request.args.get("search"),
+            request.args.get("limit", 20),
+            request.args.get("offset", 0),
+        )
+    except ValueError as exc:
+        return jsonify(error=str(exc)), 400
+    return jsonify(rows)
+
+
+async def owned_conversation(conversation_id):
+    identifier = valid_id(conversation_id)
+    if identifier is None:
+        return None
+    return await Conversation.get_or_none(id=identifier, user_id=get_jwt_identity())
+
+
+@history_blueprint.patch("/conversations/<conversation_id>/pin")
+@jwt_access_required
+async def pin(conversation_id):
+    conversation = await owned_conversation(conversation_id)
+    if conversation is None:
+        return jsonify(error="conversation not found"), 404
+    data = await request.get_json(silent=True)
+    if not isinstance(data, dict) or type(data.get("pinned")) is not bool:
+        return jsonify(error="pinned must be a boolean"), 400
+    conversation.pinned = data["pinned"]
+    # Pinning must not change conversation recency.
+    await conversation.save(update_fields=["pinned"])
+    return jsonify(id=str(conversation.id), pinned=conversation.pinned)
+
+
+async def export_owned_conversation(conversation):
+    messages = await ConversationMessage.filter(
+        conversation_id=conversation.id
+    ).order_by("created_at", "id")
+    exported = []
+    for message in messages:
+        text = display_text(message)
+        attachments = [
+            {"kind": "file", "recorded_name": name, "contents_included": False}
+            for name in re.findall(r"^📎 (.+)$", text, re.MULTILINE)
+        ]
+        if message.image_key:
+            # Stored references can be malformed or point outside this owner; never resolve or export keys.
+            attachments.append(
+                {
+                    "kind": "image",
+                    "stored_reference_present": True,
+                    "contents_included": False,
+                }
+            )
+        exported.append(
+            {
+                "id": str(message.id),
+                "speaker": message.speaker.value,
+                "created_at": message.created_at.isoformat(),
+                "text": text,
+                "provenance": (
+                    "displayed_tool_header"
+                    if message.speaker == Speaker.TOOL
+                    else "stored_message_text"
+                ),
+                "raw_tool_context_included": False,
+                "attachments": attachments,
+            }
+        )
+    return {
+        "version": 1,
+        "id": str(conversation.id),
+        "name": conversation.name,
+        "pinned": conversation.pinned,
+        "created_at": conversation.created_at.isoformat(),
+        "updated_at": conversation.updated_at.isoformat(),
+        "messages": exported,
+        "export_note": "Stored display text only. Raw tool context, artifacts, attachment contents and storage references are omitted. File names are recorded labels, not verified files.",
+    }
+
+
+@history_blueprint.get("/conversations/<conversation_id>/export")
+@jwt_access_required
+async def export(conversation_id):
+    conversation = await owned_conversation(conversation_id)
+    if conversation is None:
+        return jsonify(error="conversation not found"), 404
+    format_name = request.args.get("format", "json")
+    if format_name not in ("json", "markdown"):
+        return jsonify(error="format must be json or markdown"), 400
+    data = await export_owned_conversation(conversation)
+    if format_name == "json":
+        response = jsonify(data)
+        extension = "json"
+    else:
+        parts = [f"# {data['name']}", data["export_note"]]
+        for message in data["messages"]:
+            parts.extend(
+                [
+                    f"## {message['speaker']} — {message['created_at']}",
+                    message["text"],
+                    f"Provenance: {message['provenance']}.",
+                ]
+            )
+            if message["attachments"]:
+                parts.append(
+                    "Attachment contents omitted; recorded labels appear in the stored text."
+                )
+        response = Response(
+            "\n\n".join(parts), content_type="text/markdown; charset=utf-8"
+        )
+        extension = "md"
+    response.headers["Content-Disposition"] = (
+        f'attachment; filename="conversation-{conversation.id}.{extension}"'
+    )
+    response.headers["Cache-Control"] = "no-store"
+    return response
+
+
+def memory_row(memory):
+    return {
+        "id": str(memory.id),
+        "content": memory.content,
+        "created_at": memory.created_at.isoformat(),
+        "updated_at": memory.updated_at.isoformat(),
+    }
+
+
+@history_blueprint.get("/memories")
+@jwt_access_required
+async def memories():
+    try:
+        limit, offset = pagination(
+            request.args.get("limit", 100), request.args.get("offset", 0)
+        )
+    except ValueError as exc:
+        return jsonify(error=str(exc)), 400
+    rows = (
+        await UserMemory.filter(user_id=get_jwt_identity())
+        .order_by("-updated_at", "-id")
+        .offset(offset)
+        .limit(limit)
+    )
+    return jsonify([memory_row(row) for row in rows])
+
+
+@history_blueprint.route("/memories/<memory_id>", methods=["PATCH", "DELETE"])
+@jwt_access_required
+async def change_memory(memory_id):
+    identifier = valid_id(memory_id)
+    memory = (
+        await UserMemory.get_or_none(id=identifier, user_id=get_jwt_identity())
+        if identifier
+        else None
+    )
+    if memory is None:
+        return jsonify(error="memory not found"), 404
+    if request.method == "DELETE":
+        await memory.delete()
+        return "", 204
+    data = await request.get_json(silent=True)
+    if (
+        not isinstance(data, dict)
+        or not isinstance(data.get("content"), str)
+        or not data["content"].strip()
+        or len(data["content"]) > 10000
+    ):
+        return (
+            jsonify(error="content must be nonempty text of at most 10000 characters"),
+            400,
+        )
+    memory.content = data["content"].strip()
+    await memory.save(update_fields=["content", "updated_at"])
+    return jsonify(memory_row(memory))

blueprints/conversation/memory.py

@@ -1,3 +1,5 @@
+from collections import Counter
+
 from .models import UserMemory
 from .prompts import SIMBA_SYSTEM_PROMPT
 from .context import estimate_tokens
@@ -41,16 +43,23 @@ async def build_system_prompt_with_memories(user_id: str, query: str = "") -> st
     memories = await get_memories_for_user(user_id)
     if not memories:
         return SIMBA_SYSTEM_PROMPT
-    selected = []
     header = "\n\nUSER MEMORIES (facts the user has asked you to remember):\n"
     omission = "\n[Additional saved memories omitted from this prompt.]"
     # Reserve wrapper and omission costs so the entire injected memory block fits.
     used = estimate_tokens(header) + estimate_tokens(omission)
+    chosen = Counter()
     for memory in rank_memories(memories, query):
         cost = estimate_tokens(f"- {memory}\n")
         if used + cost <= MEMORY_BUDGET_TOKENS:
-            selected.append(memory)
+            chosen[memory] += 1
             used += cost
+    # Rank only chooses what fits; emit in stored order so the system prompt is
+    # byte-identical across queries and llama-server can reuse its KV cache.
+    selected = []
+    for memory in memories:
+        if chosen[memory]:
+            chosen[memory] -= 1
+            selected.append(memory)
     block = "\n".join(f"- {m}" for m in selected)
     if len(selected) < len(memories):
         block += omission

blueprints/conversation/models.py

@@ -23,9 +23,12 @@ class Conversation(Model):
         "models.User", related_name="conversations", null=True
     )
     channel = fields.CharField(max_length=20, default="web", null=True)
+    pinned = fields.BooleanField(default=False)
 
     class Meta:
         table = "conversations"
+        # Conversation list: a user's conversations, most recent first.
+        indexes = (("user_id", "updated_at"),)
 
 
 class ConversationMessage(Model):
@@ -41,11 +44,13 @@ class ConversationMessage(Model):
 
     class Meta:
         table = "conversation_messages"
+        # Loading a conversation's history in order.
+        indexes = (("conversation_id", "created_at"),)
 
 
 class UserMemory(Model):
     id = fields.UUIDField(primary_key=True)
-    user = fields.ForeignKeyField("models.User", related_name="memories")
+    user = fields.ForeignKeyField("models.User", related_name="memories", db_index=True)
     content = fields.TextField()
     created_at = fields.DatetimeField(auto_now_add=True)
     updated_at = fields.DatetimeField(auto_now=True)

blueprints/conversation/prompts.py

@@ -1,4 +1,6 @@
-SIMBA_SYSTEM_PROMPT = """You are Simba, Ryan's helpful personal assistant. You're named after his orange cat. You have a warm, friendly personality with a light cat-themed touch, but your priority is always being genuinely useful — give thorough, detailed answers and think things through carefully. When asked about Simba the cat, you speak as him in first person. For everything else, you're just a great assistant who happens to have a cat's name.
+SIMBA_SYSTEM_PROMPT = """You are Simba, Ryan's helpful personal assistant. You're named after his orange cat. You have a warm, friendly personality with a light cat-themed touch. When asked about Simba the cat, you speak as him in first person. For everything else, you're just a great assistant who happens to have a cat's name.
+
+Answer directly and concisely by default: lead with the answer, use short paragraphs or a compact list, and skip restating the question or adding filler sign-offs. Expand with more detail only when the user asks for it or the task genuinely calls for it, such as multi-step plans, comparisons, or explanations they requested. Be accurate: check tool results before stating numbers or dates.
 
 SIMBA FACTS (as of January 2026):
 - Name: Simba
@@ -32,46 +34,6 @@ Upcoming Appointments:
 
 IMPORTANT: When users ask factual questions about Simba's health, medical history, veterinary visits, medications, weight, or any information that would be in documents, you MUST use the simba_search tool to retrieve accurate information before answering. Do not rely on general knowledge - always search the documents for factual questions.
 
-BUDGET & FINANCE (YNAB Integration):
-You have access to Ryan's budget data through YNAB (You Need A Budget). When users ask about financial matters, use the appropriate YNAB tools:
-- Use ynab_budget_summary for a month's overview (Ready to Assign, income, spent, overspending, net worth)
-- Use ynab_account_balances for account and credit card balances
-- Use ynab_search_transactions to find specific purchases or spending at particular stores
-- Use ynab_spending_report for totals by category, payee, or account, and group_by='month' for trends
-- Use ynab_category_budget for what's assigned/available per category in a month
-- Use ynab_recent_changes for what was added, edited, or re-assigned recently
-Always use these tools when asked about budgets, spending, transactions, or financial health.
+SOURCE CITATIONS: When using retrieved documents, cite the provided Citation URLs with Markdown links. Use only URLs returned by tools; never invent source IDs or references. A citation opens the exact indexed section, which may differ from a newer original. Document date filters refer to indexed document metadata, not necessarily the date of every event described in the document.
 
-MEAL PLANNING (Mealie Integration):
-You have access to Ryan's meal plans, recipes, and shopping lists through Mealie. When users ask about food or meal planning, use the appropriate Mealie tools:
-- Use mealie_todays_meals to check what's planned for today
-- Use mealie_meal_plans to see the week's meal schedule or a custom date range
-- Use mealie_get_recipe to get full recipe details (ingredients, instructions)
-- Use mealie_shopping_lists to check what groceries are needed
-- Use mealie_create_meal_plan to add meals to the plan
-Always use these tools when asked about meals, recipes, what's for dinner, or shopping lists.
-
-HOME INVENTORY (Homebox Integration):
-If homebox_* tools are available, use them for questions about household items: where something is stored, what's in a location, warranties, and maintenance. Search before creating items to avoid duplicates, and use homebox_set_item to edit an existing item (name, description, notes, quantity, purchase info, custom fields) instead of creating a new one.
-
-WIKI (BookStack Integration):
-If bookstack_* tools are available, use them to look up and write household documentation in the BookStack wiki. Search before creating books, chapters, or pages to avoid duplicates, and prefer updating an existing page over creating a near-copy. Write page content as markdown.
-
-NOTES & RESEARCH (Obsidian Integration):
-You have access to Ryan's Obsidian vault through the Obsidian integration. When users ask about research, personal notes, or information that might be stored in markdown files, use the appropriate Obsidian tools:
-- Use obsidian_search_notes to search through your vault for relevant information
-- Use obsidian_read_note to read the full content of a specific note by path
-- Use obsidian_create_note to save new findings, ideas, or research to your vault
-- Use obsidian_create_task to create task notes with due dates
-Always use these tools when users ask about notes, research, ideas, tasks, or when you want to save information for future reference.
-
-DAILY JOURNAL (Task Tracking):
-You have access to Ryan's daily journal notes. Each note lives at 50 - Journal/YYYY/MM/YYYY-MM-DD.md and has two sections: tasks and log.
-- Use journal_get_today to read today's full daily note (tasks + log)
-- Use journal_get_tasks to list tasks (done/pending) for today or a specific date
-- Use journal_add_task to add a new task to today's (or a given date's) note
-- Use journal_complete_task to check off a task as done
-Use these tools when Ryan asks about today's tasks, wants to add something to his list, or wants to mark a task complete.
-
-USER MEMORY:
-You can remember facts about the user across conversations using the save_user_memory tool. When a user explicitly asks you to remember something, or shares a meaningful preference or personal fact, save it. Saved memories will automatically appear at the end of this prompt in future conversations under "USER MEMORIES"."""
+INTEGRATIONS: Tool descriptions cover what each tool does and its parameters; rely on those rather than this list. Use ynab_* tools for budget, spending, transaction, or financial questions. Use mealie_* tools for meals, recipes, or shopping lists. If homebox_* tools are available, use them for household items (location, warranties, maintenance); search before creating to avoid duplicates, and edit an existing item with homebox_set_item instead of creating a new one. If bookstack_* tools are available, use them for household wiki documentation; search before creating books/chapters/pages, prefer updating an existing page over a near-duplicate, and write content as markdown. Use obsidian_* tools for vault notes and research, and journal_* tools for the daily note's tasks and log (one note per day at 50 - Journal/YYYY/MM/YYYY-MM-DD.md). Use save_user_memory when the user explicitly asks you to remember something or shares a durable preference; saved memories appear below in future conversations under "USER MEMORIES"."""

blueprints/conversation/run_models.py

@@ -0,0 +1,42 @@
+"""Durable turn ownership and append-only replay log (shared PostgreSQL/SQLite)."""
+
+from tortoise import fields
+from tortoise.models import Model
+
+
+class ChatRun(Model):
+    id = fields.UUIDField(primary_key=True)
+    conversation = fields.ForeignKeyField(
+        "models.Conversation", related_name="chat_runs"
+    )
+    owner = fields.ForeignKeyField("models.User", related_name="chat_runs")
+    request_key = fields.CharField(max_length=128)
+    payload = fields.JSONField()
+    # Unique nullable slot: only one generation can write a conversation checkpoint.
+    active_key = fields.CharField(max_length=36, null=True, unique=True)
+    status = fields.CharField(max_length=20, default="running")
+    worker_token = fields.UUIDField()
+    lease_until = fields.DatetimeField(index=True)
+    cancel_requested = fields.BooleanField(default=False)
+    sequence = fields.IntField(default=0)
+    partial = fields.TextField(default="")
+    user_saved = fields.BooleanField(default=False)
+    answer_saved = fields.BooleanField(default=False)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    class Meta:
+        table = "chat_runs"
+        unique_together = (("owner", "conversation", "request_key"),)
+
+
+class ChatRunEvent(Model):
+    id = fields.BigIntField(primary_key=True)
+    run = fields.ForeignKeyField("models.ChatRun", related_name="events")
+    sequence = fields.IntField()
+    payload = fields.JSONField()
+    created_at = fields.DatetimeField(auto_now_add=True)
+
+    class Meta:
+        table = "chat_run_events"
+        unique_together = (("run", "sequence"),)

blueprints/conversation/runs.py

@@ -0,0 +1,576 @@
+"""A browser connection is only a subscriber, never the owner of generation."""
+
+import asyncio
+import json
+import logging
+import os
+import time
+import uuid
+import weakref
+from datetime import datetime, timedelta, timezone
+from contextlib import aclosing
+from contextvars import Context
+
+from quart import Blueprint, current_app, jsonify, make_response, request
+from quart_jwt_extended import get_jwt_identity
+from tortoise.exceptions import IntegrityError
+from tortoise.transactions import in_transaction
+
+from blueprints.users.decorators import jwt_access_required
+from blueprints.users.models import User
+from .models import Conversation
+from .logic import add_message_to_conversation
+from . import runtime
+from .run_models import ChatRun, ChatRunEvent
+
+runs_blueprint = Blueprint("chat_runs", __name__, url_prefix="/api/conversation")
+LEASE_SECONDS = max(10, float(os.getenv("CHAT_RUN_LEASE_SECONDS", "30")))
+MAX_SECONDS = max(1, float(os.getenv("CHAT_RUN_MAX_SECONDS", "1800")))
+_tasks = {}  # strong refs, keyed by run id; DB is the authority across processes
+# Same-process subscribers wait on these instead of polling; other processes
+# still see new events through the polling fallback.
+# Weak values: an entry lives only while some subscriber is waiting on it.
+_wakeups: weakref.WeakValueDictionary[str, asyncio.Event] = (
+    weakref.WeakValueDictionary()
+)
+# Streamed deltas are coalesced so each write carries ~this much text.
+FLUSH_SECONDS = 0.15
+POLL_SECONDS = 0.5
+_DELTA_TYPES = ("content", "thinking_delta")
+
+
+def _notify(run_id):
+    event = _wakeups.pop(str(run_id), None)
+    if event is not None:
+        event.set()
+
+
+def _waiter(run_id):
+    event = _wakeups.get(str(run_id))
+    if event is None:
+        event = _wakeups[str(run_id)] = asyncio.Event()
+    return event
+
+
+_draining_apps = set()
+
+
+def now():
+    return datetime.now(timezone.utc)
+
+
+class RunConflict(Exception):
+    def __init__(self, message, status=409):
+        super().__init__(message)
+        self.status = status
+
+
+async def owned_run(owner, conversation_id, run_id):
+    try:
+        return await ChatRun.get_or_none(
+            id=run_id,
+            owner_id=owner,
+            conversation_id=conversation_id,
+            conversation__user_id=owner,
+        )
+    except (ValueError, TypeError):
+        return None
+
+
+async def _locked(run_id, db):
+    # UPDATE takes a row lock on PostgreSQL and a write lock on SQLite, before reads.
+    await ChatRun.filter(id=run_id).using_db(db).update(updated_at=now())
+    return await ChatRun.get_or_none(id=run_id).using_db(db)
+
+
+async def _save_answer(run, text):
+    if text and not run.answer_saved:
+        conversation = await Conversation.get(id=run.conversation_id)
+        user = await User.get(id=run.owner_id)
+        await add_message_to_conversation(conversation, text, "simba", user)
+        run.answer_saved = True
+
+
+async def append(run_id, token, payload):
+    async with in_transaction() as db:
+        run = await _locked(run_id, db)
+        if (
+            run.status != "running"
+            or run.cancel_requested
+            or str(run.worker_token) != str(token)
+        ):
+            raise asyncio.CancelledError()
+        run.sequence += 1
+        if payload.get("type") == "content":
+            run.partial += payload.get("delta", "")
+        await ChatRunEvent.create(
+            run_id=run.id, sequence=run.sequence, payload=payload, using_db=db
+        )
+        await run.save(using_db=db)
+    _notify(run_id)
+
+
+async def finish(run_id, status, payload=None, token=None, tail=""):
+    """Terminalize a run; `tail` is buffered content never appended as an event."""
+    async with in_transaction() as db:
+        run = await _locked(run_id, db)
+        if (
+            run is None
+            or run.status != "running"
+            or (token and str(run.worker_token) != str(token))
+        ):
+            return
+        run.partial += tail
+        await _save_answer(run, run.partial)
+        if run.cancel_requested and status == "completed":
+            status = "cancelled"
+        run.status = status
+        run.active_key = None
+        for event in ([payload] if payload else []) + [
+            {"type": "done", "status": status}
+        ]:
+            run.sequence += 1
+            await ChatRunEvent.create(
+                run_id=run.id, sequence=run.sequence, payload=event, using_db=db
+            )
+        await run.save(using_db=db)
+    _notify(run_id)
+
+
+async def expire_runs(conversation_id=None):
+    query = ChatRun.filter(status="running", lease_until__lt=now())
+    if conversation_id:
+        query = query.filter(conversation_id=conversation_id)
+    for candidate in await query:
+        # Recheck under lock; a heartbeat may have arrived after the initial scan.
+        async with in_transaction() as db:
+            run = await _locked(candidate.id, db)
+            if run is None or run.status != "running" or run.lease_until >= now():
+                continue
+            await finish(
+                run.id,
+                "cancelled" if run.cancel_requested else "interrupted",
+                {
+                    "type": "error",
+                    "message": "Generation was interrupted by a server restart. Please retry.",
+                },
+            )
+
+
+async def create_run(app, owner, data):
+    if not isinstance(data, dict):
+        raise RunConflict("JSON object required", 400)
+    conversation_id = data.get("conversation_id")
+    try:
+        conversation = await Conversation.get_or_none(id=conversation_id, user_id=owner)
+    except (ValueError, TypeError):
+        conversation = None
+    if conversation is None:
+        raise RunConflict("conversation not found", 404)
+    key = data.get("request_id", str(uuid.uuid4()))
+    if not isinstance(key, str) or not key or len(key) > 128:
+        raise RunConflict("request_id must be 1–128 characters", 400)
+    payload = {k: data.get(k) for k in ("query", "image_key", "file_key")}
+    if not isinstance(payload["query"], str):
+        raise RunConflict("query must be a string", 400)
+    for name, prefix in (("image_key", ""), ("file_key", "files/")):
+        value = payload[name]
+        # Matches the worker's checks, so a run is never created only to fail.
+        if value and (
+            not isinstance(value, str)
+            or ".." in value
+            or not value.startswith(f"conversations/{conversation.id}/{prefix}")
+        ):
+            raise RunConflict(f"invalid {name}", 400)
+    await expire_runs(str(conversation.id))
+
+    async def existing():
+        run = await ChatRun.get_or_none(
+            owner_id=owner, conversation_id=conversation.id, request_key=key
+        )
+        if run and run.payload != payload:
+            raise RunConflict("request_id already used with different input")
+        return run
+
+    run = await existing()
+    if run:
+        return run, False
+    if id(app) in _draining_apps:
+        raise RunConflict("Server is draining; retry creation shortly", 503)
+    token = uuid.uuid4()
+    try:
+        run = await ChatRun.create(
+            owner_id=owner,
+            conversation_id=conversation.id,
+            request_key=key,
+            payload=payload,
+            active_key=str(conversation.id),
+            worker_token=token,
+            lease_until=now() + timedelta(seconds=LEASE_SECONDS),
+        )
+    except IntegrityError:
+        run = await existing()
+        if run:
+            return run, False
+        raise RunConflict("A run is already active in this conversation")
+    task = asyncio.create_task(_work(app, run.id, token), context=Context())
+    _tasks[str(run.id)] = (app, task)
+
+    def completed(task):
+        _tasks.pop(str(run.id), None)
+        if not task.cancelled() and task.exception():
+            logging.error(
+                "Chat run worker exited before finalization", exc_info=task.exception()
+            )
+
+    task.add_done_callback(completed)
+    return run, True
+
+
+class _DeltaBatch:
+    """Coalesce streamed deltas so a run writes ~FLUSH_SECONDS of text per event."""
+
+    def __init__(self, run_id, token):
+        self.run_id, self.token = run_id, token
+        self.frames: asyncio.Queue = asyncio.Queue()  # raw SSE frames not yet handled
+        self.pending = None
+        self.last_flush = 0.0
+
+    def due(self):
+        """Seconds until the pending delta must be written; None if nothing waits."""
+        if self.pending is None:
+            return None
+        return max(0.0, FLUSH_SECONDS - (time.monotonic() - self.last_flush))
+
+    def unsent_content(self):
+        """Answer text that was produced but never written as an event."""
+        parts = []
+        if self.pending is not None and self.pending["type"] == "content":
+            parts.append(self.pending["delta"])
+        while not self.frames.empty():
+            item = self.frames.get_nowait()
+            if isinstance(item, str) and item.startswith("data: {"):
+                payload = json.loads(item.removeprefix("data: "))
+                if payload.get("type") == "content":
+                    parts.append(payload.get("delta", ""))
+        return "".join(parts)
+
+    async def flush(self):
+        if self.pending is not None:
+            await append(self.run_id, self.token, self.pending)
+            self.pending = None
+            self.last_flush = time.monotonic()
+
+    async def add(self, payload):
+        if payload.get("type") not in _DELTA_TYPES:
+            await self.flush()
+            await append(self.run_id, self.token, payload)
+        elif self.pending is not None and self.pending["type"] == payload["type"]:
+            self.pending["delta"] += payload.get("delta", "")
+        else:
+            await self.flush()
+            self.pending = {**payload, "delta": payload.get("delta", "")}
+
+
+async def _heartbeat(run_id, token, worker):
+    try:
+        while True:
+            await asyncio.sleep(min(1, LEASE_SECONDS / 3))
+            changed = await ChatRun.filter(
+                id=run_id,
+                worker_token=token,
+                status="running",
+                cancel_requested=False,
+                lease_until__gte=now(),
+            ).update(lease_until=now() + timedelta(seconds=LEASE_SECONDS))
+            if not changed:
+                worker.cancel()
+                return
+    except Exception:
+        logging.exception("Chat run heartbeat failed")
+        worker.cancel()
+
+
+async def _work(app, run_id, token):
+    heartbeat = asyncio.create_task(_heartbeat(run_id, token, asyncio.current_task()))
+    batch = _DeltaBatch(run_id, token)
+    async with app.app_context():
+        try:
+            async with asyncio.timeout(MAX_SECONDS):
+                from . import _chat_run_stream
+
+                run = await ChatRun.get(id=run_id)
+                conversation = await Conversation.get(id=run.conversation_id)
+                user = await User.get(id=run.owner_id)
+
+                async def save_user(text, image_key):
+                    async with in_transaction() as db:
+                        locked = await _locked(run_id, db)
+                        if (
+                            locked is None
+                            or locked.status != "running"
+                            or locked.cancel_requested
+                        ):
+                            raise asyncio.CancelledError()
+                        if not locked.user_saved:
+                            await add_message_to_conversation(
+                                conversation, text, "user", user, image_key=image_key
+                            )
+                            locked.user_saved = True
+                            await locked.save(using_db=db)
+
+                async def save_answer(text):
+                    async with in_transaction() as db:
+                        locked = await _locked(run_id, db)
+                        if locked is not None and locked.status == "running":
+                            await _save_answer(locked, text)
+                            await locked.save(using_db=db)
+
+                failed = False
+                async with (
+                    runtime.thread_lock(str(conversation.id)),
+                    aclosing(
+                        _chat_run_stream(
+                            {**run.payload, "conversation_id": str(conversation.id)},
+                            user,
+                            conversation,
+                            save_user,
+                            save_answer,
+                        )
+                    ) as stream,
+                ):
+                    # The generator runs in one task (keeping its context across
+                    # steps) so buffered text can be flushed on a timer even while
+                    # the model emits nothing, e.g. while writing tool arguments.
+                    frames = batch.frames
+
+                    async def produce():
+                        try:
+                            async for frame in stream:
+                                frames.put_nowait(frame)
+                            frames.put_nowait(None)
+                        except BaseException as exc:
+                            frames.put_nowait(exc)
+                            raise
+
+                    producer = asyncio.create_task(produce())
+                    try:
+                        while True:
+                            try:
+                                item = await asyncio.wait_for(frames.get(), batch.due())
+                            except TimeoutError:
+                                await batch.flush()
+                                continue
+                            if item is None:
+                                break
+                            if isinstance(item, BaseException):
+                                raise item
+                            raw = item.removeprefix("data: ").strip()
+                            if raw == "[DONE]":
+                                continue
+                            payload = json.loads(raw)
+                            failed |= payload.get("type") == "error"
+                            await batch.add(payload)
+                        await batch.flush()
+                    finally:
+                        producer.cancel()
+                        await asyncio.gather(producer, return_exceptions=True)
+                await finish(run_id, "failed" if failed else "completed", token=token)
+        except asyncio.CancelledError:
+            run = await ChatRun.get_or_none(id=run_id)
+            if run is None:
+                return
+            status = "cancelled" if run.cancel_requested else "interrupted"
+            await finish(
+                run_id,
+                status,
+                {
+                    "type": "error",
+                    "message": (
+                        "Generation stopped."
+                        if status == "cancelled"
+                        else "Generation interrupted. Please retry."
+                    ),
+                },
+                token=token,
+                tail=batch.unsent_content(),
+            )
+        except Exception:
+            logging.exception("Chat generation failed")
+            await finish(
+                run_id,
+                "failed",
+                {
+                    "type": "error",
+                    "message": "Generation failed or exceeded its time limit. Please retry.",
+                },
+                token=token,
+            )
+        finally:
+            heartbeat.cancel()
+            await asyncio.gather(heartbeat, return_exceptions=True)
+
+
+async def replay_response(run, cursor):
+    async def events():
+        nonlocal cursor
+        started = time.monotonic()
+        last_expiry = 0.0
+        while time.monotonic() - started < MAX_SECONDS + LEASE_SECONDS:
+            waiter = _waiter(run.id)  # taken before reading, so no wakeup is lost
+            if time.monotonic() - last_expiry >= POLL_SECONDS:
+                await expire_runs(str(run.conversation_id))
+                last_expiry = time.monotonic()
+            batch = (
+                await ChatRunEvent.filter(run_id=run.id, sequence__gt=cursor)
+                .order_by("sequence")
+                .limit(200)
+            )
+            for event in batch:
+                cursor = event.sequence
+                if event.payload.get("type") == "done":
+                    yield f"id: {cursor}\ndata: [DONE]\n\n"
+                    return
+                yield f"id: {cursor}\ndata: {json.dumps(event.payload)}\n\n"
+            latest = await ChatRun.get_or_none(id=run.id)
+            if latest is None or (
+                latest.status != "running" and cursor >= latest.sequence
+            ):
+                yield "data: [DONE]\n\n"
+                return
+            if not batch:
+                yield ": keepalive\n\n"
+                try:
+                    await asyncio.wait_for(waiter.wait(), POLL_SECONDS)
+                except TimeoutError:
+                    pass
+        # Finite subscriber lifetime; client reattaches with its last cursor.
+
+    response = await make_response(
+        events(),
+        200,
+        {
+            "Content-Type": "text/event-stream",
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",
+            "X-Chat-Run-ID": str(run.id),
+        },
+    )
+    response.timeout = None
+    return response
+
+
+def serialize(run):
+    return {
+        "id": str(run.id),
+        "conversation_id": str(run.conversation_id),
+        "status": run.status,
+        "cursor": run.sequence,
+        "created_at": run.created_at.isoformat(),
+        "query": run.payload.get("query", ""),
+        "image_key": run.payload.get("image_key"),
+    }
+
+
+@runs_blueprint.post("/runs")
+@jwt_access_required
+async def post_run():
+    try:
+        run, created = await create_run(
+            current_app._get_current_object(),
+            get_jwt_identity(),
+            await request.get_json(),
+        )
+    except RunConflict as exc:
+        return jsonify({"error": str(exc)}), exc.status
+    return jsonify(serialize(run)), 201 if created else 200
+
+
+@runs_blueprint.get("/<conversation_id>/runs")
+@jwt_access_required
+async def discover(conversation_id):
+    owner = get_jwt_identity()
+    try:
+        conversation = await Conversation.get_or_none(id=conversation_id, user_id=owner)
+    except ValueError:
+        conversation = None
+    if conversation is None:
+        return jsonify({"error": "conversation not found"}), 404
+    await expire_runs(conversation_id)
+    # Latest terminal run included so restart/interruption can be shown after refresh.
+    run = (
+        await ChatRun.filter(conversation_id=conversation.id, owner_id=owner)
+        .order_by("-created_at")
+        .first()
+    )
+    return jsonify({"run": serialize(run) if run else None})
+
+
+@runs_blueprint.get("/<conversation_id>/runs/<run_id>/events")
+@jwt_access_required
+async def get_events(conversation_id, run_id):
+    run = await owned_run(get_jwt_identity(), conversation_id, run_id)
+    if run is None:
+        return jsonify({"error": "run not found"}), 404
+    try:
+        cursor = int(
+            request.args.get("cursor", request.headers.get("Last-Event-ID", "0"))
+        )
+        if cursor < 0 or cursor > run.sequence:
+            raise ValueError()
+    except ValueError:
+        return jsonify({"error": "invalid cursor"}), 400
+    return await replay_response(run, cursor)
+
+
+@runs_blueprint.post("/<conversation_id>/runs/<run_id>/cancel")
+@jwt_access_required
+async def cancel_run(conversation_id, run_id):
+    run = await owned_run(get_jwt_identity(), conversation_id, run_id)
+    if run is None:
+        return jsonify({"error": "run not found"}), 404
+    await ChatRun.filter(id=run.id, status="running").update(cancel_requested=True)
+    return jsonify(
+        {"status": "cancel_requested" if run.status == "running" else run.status}
+    )
+
+
+def begin_drain(app):
+    """Call when ingress/health starts draining, before awaiting run completion."""
+    _draining_apps.add(id(app))
+
+
+def is_draining(app):
+    return id(app) in _draining_apps
+
+
+async def drain_runs(app, timeout=None):
+    """Root calls BEFORE closing agents/checkpointers/database, after stopping ingress."""
+    begin_drain(app)
+    claimed_ids = {
+        run_id for run_id, (task_app, _) in list(_tasks.items()) if task_app is app
+    }
+    tasks = [task for task_app, task in list(_tasks.values()) if task_app is app]
+    if tasks:
+        _, pending = await asyncio.wait(
+            tasks,
+            timeout=(
+                timeout
+                if timeout is not None
+                else float(os.getenv("CHAT_RUN_DRAIN_SECONDS", "1800"))
+            ),
+        )
+        for task in pending:
+            task.cancel()
+        await asyncio.gather(*pending, return_exceptions=True)
+    # Covers a task cancelled before its coroutine first ran.
+    async with app.app_context():
+        for run in await ChatRun.filter(id__in=claimed_ids, status="running"):
+            await finish(
+                run.id,
+                "interrupted",
+                {
+                    "type": "error",
+                    "message": "Generation interrupted during shutdown. Please retry.",
+                },
+            )

blueprints/conversation/runtime.py

@@ -0,0 +1,186 @@
+"""Shared agent invocation for authenticated chat and messaging channels."""
+
+import asyncio
+import json
+import logging
+import time
+import uuid
+import weakref
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+
+from langchain_core.messages import convert_to_messages
+from quart import current_app
+
+from metrics import AGENT_REQUESTS_TOTAL, AGENT_RESPONSE_SECONDS
+from .checkpoints import prune_thread_checkpoints
+from .logic import get_agent_for
+from .memory import build_system_prompt_with_memories
+
+
+# Concurrent turns on one checkpoint thread race on read-modify-write of its
+# history, silently dropping a turn. Hold this around preparation + invocation.
+_thread_locks: weakref.WeakValueDictionary[str, asyncio.Lock] = (
+    weakref.WeakValueDictionary()
+)
+
+
+@asynccontextmanager
+async def thread_lock(thread_id: str):
+    lock = _thread_locks.get(thread_id)
+    if lock is None:
+        lock = _thread_locks[thread_id] = asyncio.Lock()
+    async with lock:
+        yield
+
+
+@dataclass
+class PreparedTurn:
+    agent: object
+    payload: dict
+    config: dict
+
+
+async def prepare_summary(agent, payload: dict, config: dict) -> None:
+    store = getattr(current_app, "conversation_summary_store", None)
+    if store is None:
+        return
+    configurable = config["configurable"]
+    try:
+        checkpoint = await agent.aget_state(config)
+        history = convert_to_messages(checkpoint.values.get("messages", []))
+        state = await store.update(
+            configurable["user_id"],
+            configurable["thread_id"],
+            history + convert_to_messages(payload["messages"]),
+        )
+        configurable["conversation_summary"] = state.as_message()
+    except Exception:
+        logging.exception("Failed to prepare conversation summary")
+
+
+async def prepare_agent_turn(
+    user,
+    query: str,
+    thread_id: str,
+    *,
+    channel: str = "web",
+    messages: list | None = None,
+    instructions: str = "",
+) -> PreparedTurn:
+    """Callers resolve and authorize the user/thread before preparing a turn.
+
+    Only authenticated server-side user objects determine permissions. Caller
+    input cannot supply a config that overrides identity or administration.
+    """
+    if user is None or not getattr(user, "id", None):
+        raise ValueError("An authenticated user is required for agent invocation")
+    if not isinstance(query, str) or not isinstance(thread_id, str) or not thread_id:
+        raise ValueError("query and thread_id must be valid strings")
+    prompt = await build_system_prompt_with_memories(str(user.id), query=query)
+    if instructions:
+        prompt += "\n\n" + instructions
+    config = {
+        "configurable": {
+            "user_id": str(user.id),
+            "is_admin": bool(user.is_admin()),
+            "thread_id": thread_id,
+            "system_prompt": prompt,
+            "channel": channel,
+            "request_id": str(uuid.uuid4()),
+        }
+    }
+    payload = {
+        "messages": list(messages)
+        if messages is not None
+        else [{"role": "user", "content": query}]
+    }
+    agent = get_agent_for(user)
+    await prepare_summary(agent, payload, config)
+    return PreparedTurn(agent, payload, config)
+
+
+def _record(turn: PreparedTurn, started: float, result: str) -> None:
+    cfg = turn.config["configurable"]
+    seconds = time.monotonic() - started
+    AGENT_REQUESTS_TOTAL.labels(channel=cfg["channel"], result=result).inc()
+    AGENT_RESPONSE_SECONDS.labels(channel=cfg["channel"]).observe(seconds)
+    logging.info(
+        json.dumps(
+            {
+                "event": "agent_invocation",
+                "request_id": cfg["request_id"],
+                "channel": cfg["channel"],
+                "thread_id": cfg["thread_id"],
+                "user_id": cfg["user_id"],
+                "result": result,
+                "total_ms": round(seconds * 1000),
+            }
+        )
+    )
+
+
+async def _prune_after_turn(turn: PreparedTurn) -> None:
+    checkpointer = getattr(turn.agent, "checkpointer", None)
+    if checkpointer is None:
+        return
+    try:
+        await prune_thread_checkpoints(
+            checkpointer, turn.config["configurable"]["thread_id"]
+        )
+    except Exception:
+        logging.exception("Failed to prune checkpoints")
+
+
+async def invoke_prepared(turn: PreparedTurn) -> dict:
+    started = time.monotonic()
+    result = "success"
+    try:
+        return await turn.agent.ainvoke(turn.payload, config=turn.config)
+    except asyncio.CancelledError:
+        result = "cancelled"
+        raise
+    except Exception:
+        result = "error"
+        raise
+    finally:
+        _record(turn, started, result)
+        await _prune_after_turn(turn)
+
+
+async def invoke_agent_for_user(
+    user,
+    query: str,
+    thread_id: str,
+    *,
+    channel: str = "web",
+    messages: list | None = None,
+    instructions: str = "",
+) -> dict:
+    async with thread_lock(thread_id):
+        turn = await prepare_agent_turn(
+            user,
+            query,
+            thread_id,
+            channel=channel,
+            messages=messages,
+            instructions=instructions,
+        )
+        return await invoke_prepared(turn)
+
+
+async def stream_prepared(turn: PreparedTurn):
+    started = time.monotonic()
+    result = "cancelled"
+    try:
+        async for event in turn.agent.astream_events(
+            turn.payload, version="v2", config=turn.config
+        ):
+            yield event
+        result = "success"
+    except Exception:
+        result = "error"
+        raise
+    finally:
+        _record(turn, started, result)
+        await _prune_after_turn(turn)

blueprints/conversation/tool_routing.py

@@ -1,4 +1,8 @@
-"""Deterministic per-call routing; discovery grants live in tool messages, not globals."""
+"""Tool selection per model call.
+
+Static mode (the default) binds the full role-appropriate set. Dynamic mode
+routes per call by keyword; discovery grants live in tool messages, not globals.
+"""
 
 import json
 import re
@@ -29,6 +33,7 @@ ADMIN_TOOL_NAMES = {
 }
 CORE_TOOL_NAMES = {"get_current_date", "save_user_memory"}
 DISCOVERY_NAME = "discover_tool_groups"
+TOOL_ROUTING_MODES = ("static", "dynamic")
 
 
 def words(text):
@@ -138,6 +143,16 @@ class ToolRouter:
         )
         return discover_tool_groups
 
+    def select_static(self, *, is_admin=False):
+        """Every role-appropriate tool in registration order, without discovery.
+
+        The bound set never varies by query or turn, so the rendered tool block
+        stays byte-identical and llama-server can reuse its cached prefix.
+        """
+        return [
+            t for t in self.tools if is_admin or tool_name(t) not in self.admin_names
+        ]
+
     def select(self, messages, *, is_admin=False):
         available = self.available_groups(is_admin)
         human_indices = [

blueprints/cron/__init__.py

@@ -1,9 +1,9 @@
 import hmac
 import logging
 import os
-from datetime import datetime, timezone
 
-from quart import Blueprint, current_app, jsonify, request
+from blueprints.scheduled_messages.validation import json_object
+from quart import Blueprint, jsonify, request
 
 from blueprints.users.decorators import admin_required
 from blueprints.scheduled_messages.models import MessageChannel
@@ -23,6 +23,7 @@ def _serialize(job: CronJob) -> dict:
         "recipient": job.recipient,
         "channel": job.channel.value,
         "enabled": job.enabled,
+        "created_by": str(job.created_by_id) if job.created_by_id else None,
         "last_run_at": job.last_run_at.isoformat() if job.last_run_at else None,
         "last_status": job.last_status,
         "last_error": job.last_error,
@@ -49,9 +50,12 @@ async def list_jobs():
 @cron_blueprint.route("/", methods=["POST"])
 @admin_required
 async def create_job():
-    data = await request.get_json()
-    if not data:
-        return jsonify({"error": "Invalid payload"}), 400
+    data, error = await json_object(
+        ("name", "prompt", "recipient", "channel", "description", "created_by"),
+        nullable=("description", "created_by"),
+    )
+    if error:
+        return error
 
     name = (data.get("name") or "").strip()
     prompt = (data.get("prompt") or "").strip()
@@ -71,7 +75,10 @@ async def create_job():
             {"error": f"Invalid channel: {channel_str}. Must be 'imessage' or 'email'"}
         ), 400
 
+    from quart_jwt_extended import get_jwt_identity
+
     job = await CronJob.create(
+        created_by_id=get_jwt_identity(),
         name=name,
         description=description,
         prompt=prompt,
@@ -89,10 +96,30 @@ async def update_job(job_id: str):
     if not job:
         return jsonify({"error": "Not found"}), 404
 
-    data = await request.get_json()
-    if not data:
-        return jsonify({"error": "Invalid payload"}), 400
+    data, error = await json_object(
+        ("name", "prompt", "recipient", "channel", "description", "created_by"),
+        nullable=("description", "created_by"),
+    )
+    if error:
+        return error
 
+    blank = [
+        f for f in ("name", "prompt", "recipient") if f in data and not data[f].strip()
+    ]
+    if blank:
+        return jsonify({"error": f"{', '.join(blank)} cannot be empty"}), 400
+
+    if "created_by" in data:
+        from blueprints.users.models import User
+
+        owner = (
+            await User.get_or_none(id=data["created_by"])
+            if data["created_by"]
+            else None
+        )
+        if not owner:
+            return jsonify({"error": "created_by must identify an existing user"}), 400
+        job.created_by_id = owner.id
     if "name" in data:
         job.name = data["name"].strip()
     if "description" in data:
@@ -133,44 +160,49 @@ async def trigger_job(job_id: str):
     if not job:
         return jsonify({"error": "Not found"}), 404
 
-    if not job.enabled:
-        return jsonify({"skipped": True}), 200
+    data = await request.get_json(silent=True)
+    if not isinstance(data, dict):
+        data = {}
+    run_id = request.headers.get("Idempotency-Key") or data.get("run_id")
+    if not isinstance(run_id, str) or not run_id.strip() or len(run_id) > 255:
+        return jsonify(
+            {"error": "run_id or Idempotency-Key is required (maximum 255 characters)"}
+        ), 400
+    from .scheduler import execute_job
 
-    try:
-        import uuid
+    result = await execute_job(job, run_id)
+    return jsonify(result), 200 if result.get("ok") or result.get("skipped") else 409
 
-        response = await current_app.main_agent.ainvoke(
-            {"messages": [{"role": "user", "content": job.prompt}]},
-            config={"configurable": {"thread_id": f"cron-{uuid.uuid4()}"}},
-        )
-        result = response.get("messages", [])[-1].content
 
-        if job.channel == MessageChannel.IMESSAGE:
-            from blueprints.imessage import send_imessage
-            from utils.strip_markdown import strip_markdown
+@cron_blueprint.route("/<job_id>/deliveries", methods=["GET"])
+@admin_required
+async def delivery_history(job_id):
+    if not await CronJob.filter(id=job_id).exists():
+        return jsonify({"error": "Not found"}), 404
+    from blueprints.scheduled_messages.delivery import history
 
-            await send_imessage(job.recipient, strip_markdown(result))
+    return jsonify(await history("cron", job_id))
 
-        elif job.channel == MessageChannel.EMAIL:
-            from blueprints.email import send_email_reply
 
-            await send_email_reply(
-                to=job.recipient,
-                subject=job.name,
-                body=result,
-            )
+@cron_blueprint.route("/<job_id>/retry", methods=["POST"])
+@admin_required
+async def retry_job(job_id):
+    job = await CronJob.get_or_none(id=job_id)
+    if not job:
+        return jsonify({"error": "Not found"}), 404
+    data = await request.get_json(silent=True)
+    if not isinstance(data, dict):
+        data = {}
+    run_id = data.get("run_id")
+    if not isinstance(run_id, str) or not run_id or len(run_id) > 255:
+        return jsonify({"error": "run_id is required"}), 400
+    from blueprints.scheduled_messages.models import JobDelivery
 
-        job.last_run_at = datetime.now(timezone.utc)
-        job.last_status = "success"
-        job.last_error = None
-        await job.save()
-        logger.info(f"Cron job {job.id} ({job.name}) executed successfully")
-        return jsonify({"ok": True})
+    if not await JobDelivery.filter(
+        kind="cron", job_id=str(job.id), run_id=run_id, status="failed"
+    ).exists():
+        return jsonify({"error": "Only safe failed deliveries can be retried"}), 409
+    from .scheduler import execute_job
 
-    except Exception as e:
-        job.last_run_at = datetime.now(timezone.utc)
-        job.last_status = "failed"
-        job.last_error = str(e)
-        await job.save()
-        logger.error(f"Cron job {job.id} ({job.name}) failed: {e}")
-        return jsonify({"error": str(e)}), 500
+    result = await execute_job(job, run_id, retry=True)
+    return jsonify(result), 200 if result.get("ok") or result.get("skipped") else 409

blueprints/cron/models.py

@@ -6,6 +6,9 @@ from blueprints.scheduled_messages.models import MessageChannel
 
 class CronJob(Model):
     id = fields.UUIDField(primary_key=True)
+    created_by = fields.ForeignKeyField(
+        "models.User", null=True, related_name="cron_jobs", on_delete=fields.SET_NULL
+    )
     name = fields.CharField(max_length=255)
     description = fields.TextField(null=True)
     prompt = fields.TextField()

blueprints/cron/scheduler.py

@@ -0,0 +1,59 @@
+"""One durable delivery per caller-supplied cron run identifier."""
+
+from tortoise.transactions import in_transaction
+
+from blueprints.scheduled_messages import delivery
+from .models import CronJob
+
+
+async def execute_job(job, run_id, *, retry=False):
+    await delivery.expire_claims("cron")
+    if not job.enabled:
+        return {"skipped": True, "status": "disabled"}
+    claimed = await delivery.claim("cron", job.id, run_id, retry=retry)
+    if not claimed:
+        from blueprints.scheduled_messages.models import JobDelivery
+
+        existing = await JobDelivery.get(kind="cron", job_id=str(job.id), run_id=run_id)
+        return {"skipped": True, "status": existing.status}
+    sending = False
+    try:
+        if not job.created_by_id:
+            raise ValueError(
+                "Cron job requires an explicit created_by owner before execution"
+            )
+        user = await job.created_by
+        if user is None:
+            raise ValueError("Cron job owner no longer exists")
+        content = await delivery.agent_content(
+            user, job.prompt, f"cron-{claimed.id}", "cron"
+        )
+        async with in_transaction() as db:
+            active = (
+                await CronJob.filter(id=job.id, enabled=True)
+                .using_db(db)
+                .update(updated_at=delivery.now())
+            )
+            if not active:
+                await delivery.transition(claimed, "cancelled")
+                return {"skipped": True, "status": "cancelled"}
+            if not await delivery.transition(claimed, "sending"):
+                return {"skipped": True, "status": "uncertain"}
+        sending = True
+        await delivery.send(job.channel, job.recipient, content, job.name)
+        await delivery.transition(claimed, "sent")
+        status, error = "sent", None
+    except Exception as exc:
+        status = (
+            "uncertain"
+            if sending and not isinstance(exc, delivery.SafeDeliveryError)
+            else "failed"
+        )
+        error = str(exc)
+        await delivery.transition(claimed, status, error)
+    await CronJob.filter(id=job.id).update(
+        last_run_at=delivery.now(),
+        last_status="success" if status == "sent" else status,
+        last_error=error,
+    )
+    return {"ok": status == "sent", "status": status, "error": error}

blueprints/email/__init__.py

@@ -7,7 +7,7 @@ import time
 from collections import defaultdict
 
 import httpx
-from quart import Blueprint, current_app, request
+from quart import Blueprint, request
 
 from blueprints.users.models import User
 from blueprints.conversation.logic import (
@@ -15,8 +15,8 @@ from blueprints.conversation.logic import (
     add_message_to_conversation,
     persist_agent_tool_messages,
 )
-from blueprints.conversation.models import Speaker
-from blueprints.conversation.prompts import SIMBA_SYSTEM_PROMPT
+from blueprints.conversation.runtime import invoke_agent_for_user
+from utils.strip_markdown import extract_agent_text
 from . import models  # noqa: F401 — register Tortoise ORM models
 from .helpers import generate_email_token, get_user_email_address  # noqa: F401
 
@@ -193,18 +193,14 @@ async def webhook():
 
     # Build messages payload
     try:
-        messages = await conversation.messages.all()
-        recent_messages = list(messages)[-20:]
-
-        messages_payload = [{"role": "system", "content": SIMBA_SYSTEM_PROMPT}]
-        for msg in recent_messages[:-1]:
-            role = "user" if msg.speaker == Speaker.USER else "assistant"
-            messages_payload.append({"role": role, "content": msg.text})
-        messages_payload.append({"role": "user", "content": body})
-
-        logger.info(f"Invoking LangChain agent with {len(messages_payload)} messages")
-        response = await current_app.main_agent.ainvoke({"messages": messages_payload})
-        response_text = response.get("messages", [])[-1].content
+        response = await invoke_agent_for_user(
+            user,
+            body,
+            str(conversation.id),
+            channel="email",
+            instructions="You are replying by email. Write a clear reply appropriate to that channel.",
+        )
+        response_text = extract_agent_text(response.get("messages", [])[-1].content)
         await persist_agent_tool_messages(
             conversation, user, response.get("messages", [])
         )

blueprints/email/helpers.py

@@ -4,9 +4,9 @@ import hashlib
 
 def generate_email_token(user_id: str, secret: str) -> str:
     """Generate a 16-char hex HMAC token for a user's email address."""
-    return hmac.new(
-        secret.encode(), str(user_id).encode(), hashlib.sha256
-    ).hexdigest()[:16]
+    return hmac.new(secret.encode(), str(user_id).encode(), hashlib.sha256).hexdigest()[
+        :16
+    ]
 
 
 def get_user_email_address(token: str, domain: str) -> str:

blueprints/imessage/__init__.py

@@ -11,13 +11,12 @@ from quart import Blueprint, request, jsonify
 
 from blueprints.users.models import User
 from blueprints.conversation.logic import (
-    get_agent_for,
     get_conversation_for_channel,
     add_message_to_conversation,
     persist_agent_tool_messages,
 )
 from blueprints.conversation.models import Conversation, ConversationMessage
-from blueprints.conversation.memory import build_system_prompt_with_memories
+from blueprints.conversation.runtime import invoke_agent_for_user
 from utils.strip_markdown import extract_agent_text, strip_markdown
 
 imessage_blueprint = Blueprint("imessage_api", __name__, url_prefix="/api/imessage")
@@ -327,7 +326,7 @@ async def _process_and_reply(
             async with httpx.AsyncClient() as client:
                 resp = await client.get(media_url, timeout=30)
                 resp.raise_for_status()
-            image_block = image_content_block(resp.content)
+            image_block = await asyncio.to_thread(image_content_block, resp.content)
         except Exception as e:
             logger.error(f"Failed to load iMessage image: {e}")
             content = f"{content or ''}\n\n[An image was sent but could not be loaded.]".strip()
@@ -344,24 +343,13 @@ async def _process_and_reply(
         else:
             user_content = content
 
-        system_prompt = await build_system_prompt_with_memories(str(user.id))
-        system_prompt += (
-            "\n\nIMPORTANT: You are replying via iMessage. Use plain text only. "
-            "Do not use markdown tables, headers, bullet points, bold, italics, "
-            "or any other markdown formatting. Write in natural prose."
-        )
-        agent_config = {
-            "configurable": {
-                "user_id": str(user.id),
-                "is_admin": user.is_admin(),
-                "thread_id": str(conversation.id),
-                "system_prompt": system_prompt,
-            }
-        }
-        logger.info(f"Invoking LangChain agent for conversation {conversation.id}")
-        response = await get_agent_for(user).ainvoke(
-            {"messages": [{"role": "user", "content": user_content}]},
-            config=agent_config,
+        response = await invoke_agent_for_user(
+            user,
+            content or "",
+            str(conversation.id),
+            channel="imessage",
+            messages=[{"role": "user", "content": user_content}],
+            instructions="You are replying via iMessage. Use plain text and natural prose; avoid markdown formatting.",
         )
         response_text = extract_agent_text(response.get("messages", [])[-1].content)
         await persist_agent_tool_messages(

blueprints/integrations/__init__.py

@@ -0,0 +1,116 @@
+"""Authenticated, secret-free status of configured integrations and cached data."""
+
+import asyncio
+import os
+from datetime import datetime, timezone
+
+from quart import Blueprint, jsonify
+from quart_jwt_extended import get_jwt_identity
+
+from blueprints.users.decorators import jwt_access_required
+from blueprints.users.models import User
+from blueprints.ynab.sync import _current_state, ynab_configured
+
+integrations_blueprint = Blueprint(
+    "integrations_api", __name__, url_prefix="/api/integrations"
+)
+
+
+def configuration_status(name: str, configured: bool) -> dict:
+    return {
+        "name": name,
+        "status": "configured" if configured else "not_configured",
+        "last_success_at": None,
+        "message": "Configured; connection has not been checked."
+        if configured
+        else "Not configured.",
+    }
+
+
+async def _ynab_status() -> dict:
+    status = configuration_status("YNAB", ynab_configured())
+    if not ynab_configured():
+        return status
+    try:
+        state = await _current_state()
+        if state is None or state.last_synced_at is None:
+            return {
+                **status,
+                "status": "unavailable",
+                "message": "No synchronized budget data is available.",
+            }
+        stamp = state.last_synced_at
+        if stamp.tzinfo is None:
+            stamp = stamp.replace(tzinfo=timezone.utc)
+        age = max(0, (datetime.now(timezone.utc) - stamp).total_seconds())
+        stale = age > 120 or bool(state.last_error)
+        return {
+            **status,
+            "status": "stale" if stale else "ready",
+            "last_success_at": stamp.isoformat(),
+            "message": "Using cached budget data; synchronization is overdue or failed."
+            if stale
+            else "Budget data is current.",
+        }
+    except Exception:
+        return {
+            **status,
+            "status": "unavailable",
+            "message": "Budget synchronization status is unavailable.",
+        }
+
+
+async def integration_statuses(*, is_admin: bool) -> list[dict]:
+    result = [await _ynab_status()]
+    notes = configuration_status("Document search", bool(os.getenv("DATABASE_URL")))
+    try:
+        from blueprints.rag.logic import get_vector_store_stats
+
+        stats = await asyncio.to_thread(get_vector_store_stats)
+        count = stats.get("total_documents", 0)
+        notes.update(
+            status="ready" if count else "unavailable",
+            indexed_chunks=count,
+            message=f"{count} indexed document chunks available."
+            if count
+            else "No documents are indexed yet.",
+        )
+    except Exception:
+        notes.update(
+            status="unavailable", message="Document index status is unavailable."
+        )
+    result.append(notes)
+    for name, keys in [
+        ("Mealie", ("MEALIE_BASE_URL", "MEALIE_API_TOKEN")),
+        ("Obsidian", ("OBSIDIAN_VAULT_PATH",)),
+        ("Web search", ("TAVILY_API_KEY",)),
+        ("Local model", ("LLAMA_SERVER_URL",)),
+        ("Fallback model", ("OPENAI_API_KEY",)),
+        ("SimbaKit", ("SIMBAKIT_URL",)),
+    ]:
+        result.append(configuration_status(name, all(os.getenv(k) for k in keys)))
+    if is_admin:
+        for name, keys in [
+            ("Homebox", ("HOMEBOX_URL", "HOMEBOX_TOKEN")),
+            (
+                "BookStack",
+                ("BOOKSTACK_BASE_URL", "BOOKSTACK_TOKEN_ID", "BOOKSTACK_TOKEN_SECRET"),
+            ),
+            ("Google", ("GOOGLE_TOKEN_FILE",)),
+            ("iMessage", ("SENDBLUE_API_KEY", "SENDBLUE_API_SECRET")),
+            ("Email", ("MAILGUN_API_KEY", "MAILGUN_DOMAIN")),
+        ]:
+            result.append(configuration_status(name, all(os.getenv(k) for k in keys)))
+    return result
+
+
+@integrations_blueprint.get("/")
+@jwt_access_required
+async def status():
+    user = await User.get(id=get_jwt_identity())
+    return jsonify(
+        {
+            "checked_at": datetime.now(timezone.utc).isoformat(),
+            "integrations": await integration_statuses(is_admin=user.is_admin()),
+        }
+    )

blueprints/rag/logic.py

@@ -1,4 +1,6 @@
+import asyncio
 import datetime
+import json
 import logging
 import os
 import re
@@ -16,6 +18,7 @@ from langchain_text_splitters import (
 from sqlalchemy import create_engine, text
 
 from .fetchers import PaperlessNGXService
+from .retrieval import build_filter, citation_url, document_date, document_dates
 from utils.obsidian_service import ObsidianService
 
 # Load environment variables
@@ -67,6 +70,9 @@ def _get_engine():
     return _get_engine._engine
 
 
+# Chunks per embedding request when indexing Paperless documents.
+EMBED_BATCH_SIZE = 32
+
 text_splitter = RecursiveCharacterTextSplitter(
     chunk_size=1000,  # chunk size (characters)
     chunk_overlap=200,  # chunk overlap (characters)
@@ -139,6 +145,7 @@ def date_to_epoch(date_str: str) -> float:
         0,
         0,
         0,
+        tzinfo=datetime.timezone.utc,
     )
 
     return date.timestamp()
@@ -152,6 +159,9 @@ async def fetch_documents_from_paperless_ngx() -> list[Document]:
     for doc in data:
         metadata = {
             "source": "paperless",
+            "document_id": doc.get("id"),
+            "document_date": doc["created_date"][:10],
+            "indexed_at": time.time(),
             "created_date": date_to_epoch(doc["created_date"]),
             "filename": doc["original_file_name"],
             "document_type": doctypes.get(doc["document_type"], ""),
@@ -200,15 +210,25 @@ async def index_documents():
     splits = _sanitize_documents(splits)
     logger.info(f"Indexing {len(splits)} chunks from {len(documents)} documents")
     vector_store = _get_vector_store()
-    for i, split in enumerate(splits):
+    for start in range(0, len(splits), EMBED_BATCH_SIZE):
+        batch = splits[start : start + EMBED_BATCH_SIZE]
         try:
-            await vector_store.aadd_documents(documents=[split])
-        except Exception as e:
-            logger.error(
-                f"Failed to embed chunk {i} from {split.metadata.get('filename', 'unknown')}: {e}"
+            await vector_store.aadd_documents(documents=batch)
+        except Exception as batch_error:
+            # Embedding fails before anything is inserted; retry the batch one
+            # chunk at a time to name the chunk the embedding server rejects.
+            logger.warning(
+                f"Batch of {len(batch)} chunks failed ({batch_error}); retrying one at a time"
             )
-            logger.debug(f"Chunk content preview: {split.page_content[:200]!r}")
-            raise
+            for i, split in enumerate(batch, start):
+                try:
+                    await vector_store.aadd_documents(documents=[split])
+                except Exception as e:
+                    logger.error(
+                        f"Failed to embed chunk {i} from {split.metadata.get('filename', 'unknown')}: {e}"
+                    )
+                    logger.debug(f"Chunk content preview: {split.page_content[:200]!r}")
+                    raise
 
 
 async def fetch_obsidian_documents() -> list[Document]:
@@ -307,6 +327,57 @@ def _load_obsidian_index_cache() -> dict[str, float]:
         return {row[0]: row[1] for row in result if row[0] is not None}
 
 
+def _walk_vault_mtimes(obsidian_service: ObsidianService) -> dict[str, float]:
+    """Walk the vault and stat every file. Blocking; run via asyncio.to_thread."""
+    return {
+        str(md_path): md_path.stat().st_mtime
+        for md_path in obsidian_service.walk_vault()
+    }
+
+
+def _read_obsidian_document(
+    obsidian_service: ObsidianService, filepath: str, now: float
+) -> Document | None:
+    """Read and parse one vault file into a Document, or None on error."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            content = f.read()
+        parsed = obsidian_service.parse_markdown(content, filepath)
+        metadata = {
+            "source": "obsidian",
+            "filepath": parsed["filepath"],
+            "folder": str(Path(parsed["filepath"]).parent)
+            if parsed["filepath"]
+            else "",
+            "tags": parsed["tags"],
+            "created_at": parsed["metadata"].get("created_at"),
+            "indexed_at": now,
+            **{
+                k: v
+                for k, v in parsed["metadata"].items()
+                if k not in ["created_at", "created_by"]
+            },
+        }
+        return Document(
+            page_content=parsed["content"], metadata=_make_serializable(metadata)
+        )
+    except Exception as e:
+        logger.warning(f"Error reading {filepath}: {e}")
+        return None
+
+
+def _read_obsidian_documents(
+    obsidian_service: ObsidianService, filepaths: list[str], now: float
+) -> list[Document]:
+    """Read and parse a batch of vault files. Blocking; run via asyncio.to_thread."""
+    documents = []
+    for filepath in filepaths:
+        document = _read_obsidian_document(obsidian_service, filepath, now)
+        if document is not None:
+            documents.append(document)
+    return documents
+
+
 async def sync_obsidian_documents() -> dict[str, int]:
     """Incrementally sync Obsidian documents to pgvector.
 
@@ -322,69 +393,52 @@ async def sync_obsidian_documents() -> dict[str, int]:
 
     # Load cache from DB on first run
     if not _obsidian_index_cache:
-        _obsidian_index_cache = _load_obsidian_index_cache()
+        _obsidian_index_cache = await asyncio.to_thread(_load_obsidian_index_cache)
 
     # Build map of current vault files -> mtime
-    vault_files: dict[str, float] = {}
-    for md_path in obsidian_service.walk_vault():
-        vault_files[str(md_path)] = md_path.stat().st_mtime
+    vault_files = await asyncio.to_thread(_walk_vault_mtimes, obsidian_service)
 
     added = 0
     updated = 0
-    deleted = 0
 
     # Find files to add or update
     files_to_index: list[str] = []
+    stale_filepaths: list[str] = []
     for filepath, mtime in vault_files.items():
         indexed_at = _obsidian_index_cache.get(filepath)
         if indexed_at is None:
             files_to_index.append(filepath)
             added += 1
         elif mtime > indexed_at:
-            # Delete old chunks first
-            delete_documents_by_metadata("filepath", filepath)
+            stale_filepaths.append(filepath)
             files_to_index.append(filepath)
             updated += 1
 
-    # Find deleted files (in cache but not on disk)
-    for filepath in list(_obsidian_index_cache):
-        if filepath not in vault_files:
-            delete_documents_by_metadata("filepath", filepath)
-            del _obsidian_index_cache[filepath]
-            deleted += 1
+    # Find deleted files (in cache but not on disk). Don't drop them from the
+    # cache yet: if the delete below raises, we want the next sync to still
+    # know about them and retry, rather than leaving their chunks searchable
+    # forever with no record that they need cleaning up.
+    removed_filepaths = [fp for fp in _obsidian_index_cache if fp not in vault_files]
+    stale_filepaths.extend(removed_filepaths)
+    deleted = len(removed_filepaths)
+
+    # Delete old chunks for updated/removed files before re-indexing, in one
+    # transaction instead of a round-trip per file. Left uncaught: a failure
+    # here must stop the sync so the cache isn't marked clean for work that
+    # didn't happen.
+    if stale_filepaths:
+        await asyncio.to_thread(delete_documents_by_filepaths, stale_filepaths)
+
+    # Only now that the delete has succeeded is it safe to forget these files.
+    for filepath in removed_filepaths:
+        del _obsidian_index_cache[filepath]
 
     # Index new/changed files
     if files_to_index:
         now = time.time()
-        documents = []
-        for filepath in files_to_index:
-            try:
-                with open(filepath, "r", encoding="utf-8") as f:
-                    content = f.read()
-                parsed = obsidian_service.parse_markdown(content, filepath)
-                metadata = {
-                    "source": "obsidian",
-                    "filepath": parsed["filepath"],
-                    "folder": str(Path(parsed["filepath"]).parent)
-                    if parsed["filepath"]
-                    else "",
-                    "tags": parsed["tags"],
-                    "created_at": parsed["metadata"].get("created_at"),
-                    "indexed_at": now,
-                    **{
-                        k: v
-                        for k, v in parsed["metadata"].items()
-                        if k not in ["created_at", "created_by"]
-                    },
-                }
-                document = Document(
-                    page_content=parsed["content"],
-                    metadata=_make_serializable(metadata),
-                )
-                documents.append(document)
-            except Exception as e:
-                logger.warning(f"Error reading {filepath}: {e}")
-                continue
+        documents = await asyncio.to_thread(
+            _read_obsidian_documents, obsidian_service, files_to_index, now
+        )
 
         if documents:
             splits = []
@@ -409,6 +463,8 @@ async def query_vector_store(
     query: str,
     source: str | None = None,
     k: int = 8,
+    start_date: str | None = None,
+    end_date: str | None = None,
 ):
     """Query the vector store with optional source filtering and MMR.
 
@@ -417,21 +473,62 @@ async def query_vector_store(
         source: Filter by source metadata (e.g., "obsidian", "paperless")
         k: Number of results to return
     """
+    if not isinstance(query, str) or not query.strip():
+        raise ValueError("query must not be empty")
+    if not isinstance(k, int) or not 1 <= k <= 20:
+        raise ValueError("k must be between 1 and 20")
+    filter_dict = build_filter(source, start_date, end_date)
     vector_store = _get_vector_store()
-    filter_dict = {"source": source} if source else None
     retrieved_docs = await vector_store.amax_marginal_relevance_search(
         query,
         k=k,
         fetch_k=k * 3,
         filter=filter_dict,
     )
+    if start_date or end_date:
+        retrieved_docs = [
+            doc
+            for doc in retrieved_docs
+            # Same "any date field in range" rule as the database filter.
+            if any(
+                (not start_date or stamp >= start_date)
+                and (not end_date or stamp <= end_date)
+                for stamp in document_dates(doc.metadata)
+            )
+        ]
     serialized = "\n\n".join(
-        (f"Source: {doc.metadata}\nContent: {doc.page_content}")
+        (
+            f"{_chunk_header(doc)}\n"
+            + (f"Citation: {citation_url(doc)}\n" if citation_url(doc) else "")
+            + f"Content: {doc.page_content}"
+        )
         for doc in retrieved_docs
     )
     return serialized, retrieved_docs
 
 
+def _chunk_title(doc: Document) -> str:
+    from blueprints.conversation.artifacts_util import note_title, vault_relative_path
+
+    meta = doc.metadata or {}
+    if meta.get("source") == "obsidian":
+        path = vault_relative_path(meta.get("filepath") or "")
+        return note_title(path, meta) if path else "Untitled note"
+    filename = meta.get("filename") or ""
+    return Path(filename).stem or filename or "Document"
+
+
+def _chunk_header(doc: Document) -> str:
+    """Compact per-chunk header (title, date, source) replacing the raw metadata dict."""
+    meta = doc.metadata or {}
+    parts = [f"Title: {_chunk_title(doc)}"]
+    if date := document_date(meta):
+        parts.append(f"Date: {date}")
+    if source := meta.get("source"):
+        parts.append(f"Source: {source}")
+    return " | ".join(parts)
+
+
 def delete_all_documents():
     """Delete all documents from the vector store collection."""
     collection_id = _get_collection_id()
@@ -446,6 +543,27 @@ def delete_all_documents():
         conn.commit()
 
 
+def _delete_by_containment(conn, collection_id, filter_dict: dict) -> None:
+    """Delete rows whose cmetadata contains filter_dict.
+
+    Uses JSONB containment (`@>`) rather than `cmetadata->>key = value` so the
+    delete can use the GIN index langchain_postgres creates on cmetadata
+    (ix_cmetadata_gin, jsonb_path_ops); `->>` text extraction can't use it.
+
+    Unlike `->>`, which coerces to text before comparing, `@>` matches on the
+    stored JSON type: the string "123" does not match the number 123. Callers
+    must pass values with the same type as what was stored (today's only
+    caller, filepath/source strings, is fine).
+    """
+    conn.execute(
+        text(
+            "DELETE FROM langchain_pg_embedding "
+            "WHERE collection_id = :cid AND cmetadata @> CAST(:filter AS jsonb)"
+        ),
+        {"cid": collection_id, "filter": json.dumps(filter_dict)},
+    )
+
+
 def delete_documents_by_metadata(key: str, value: str):
     """Delete documents matching a metadata key/value pair."""
     collection_id = _get_collection_id()
@@ -453,13 +571,25 @@ def delete_documents_by_metadata(key: str, value: str):
         return
     engine = _get_engine()
     with engine.connect() as conn:
-        conn.execute(
-            text(
-                "DELETE FROM langchain_pg_embedding "
-                "WHERE collection_id = :cid AND cmetadata->>:key = :value"
-            ),
-            {"cid": collection_id, "key": key, "value": value},
-        )
+        _delete_by_containment(conn, collection_id, {key: value})
+        conn.commit()
+
+
+def delete_documents_by_filepaths(filepaths: list[str]) -> None:
+    """Delete all chunks for several obsidian filepaths in one transaction.
+
+    One containment delete per path, committed together, so a sync pass with
+    many stale files doesn't pay a commit round-trip per file.
+    """
+    if not filepaths:
+        return
+    collection_id = _get_collection_id()
+    if not collection_id:
+        return
+    engine = _get_engine()
+    with engine.connect() as conn:
+        for filepath in filepaths:
+            _delete_by_containment(conn, collection_id, {"filepath": filepath})
         conn.commit()
 
 

blueprints/rag/retrieval.py

@@ -0,0 +1,105 @@
+"""Validated retrieval filters and stable, authenticated source references."""
+
+from datetime import date, datetime, time, timedelta, timezone
+import re
+from uuid import UUID
+
+
+def parse_date(value: str | None) -> date | None:
+    if not value:
+        return None
+    try:
+        if not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
+            raise ValueError
+        return date.fromisoformat(value)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("Dates must be YYYY-MM-DD.") from exc
+
+
+def build_filter(
+    source: str | None, start_date: str | None = None, end_date: str | None = None
+) -> dict | None:
+    if source not in {None, "paperless", "obsidian"}:
+        raise ValueError("source must be paperless or obsidian")
+    start, end = parse_date(start_date), parse_date(end_date)
+    if start and end and start > end:
+        raise ValueError("start_date must be on or before end_date")
+    parts = [{"source": source}] if source else []
+    if start or end:
+        strings, epochs = {}, {}
+        if start:
+            strings["$gte"] = start.isoformat()
+            epochs["$gte"] = datetime.combine(start, time.min, timezone.utc).timestamp()
+        if end:
+            try:
+                next_day = end + timedelta(days=1)
+            except OverflowError as exc:
+                raise ValueError("end_date is beyond the supported date range") from exc
+            strings["$lt"] = next_day.isoformat()
+            epochs["$lt"] = datetime.combine(
+                next_day, time.min, timezone.utc
+            ).timestamp()
+
+        # Both newly normalized fields and existing indexed metadata remain usable.
+        def date_bounds(field, bounds):
+            clauses = [{field: {operator: value}} for operator, value in bounds.items()]
+            return {"$and": clauses} if len(clauses) > 1 else clauses[0]
+
+        parts.append(
+            {
+                "$or": [
+                    date_bounds("document_date", strings),
+                    {
+                        "$and": [
+                            {"source": "paperless"},
+                            date_bounds("created_date", epochs),
+                        ]
+                    },
+                    {"$and": [{"source": "obsidian"}, date_bounds("date", strings)]},
+                    {
+                        "$and": [
+                            {"source": "obsidian"},
+                            date_bounds("created_at", strings),
+                        ]
+                    },
+                ]
+            }
+        )
+    return {"$and": parts} if len(parts) > 1 else parts[0] if parts else None
+
+
+def document_dates(metadata: dict) -> list[str]:
+    """Every date the search filter can match on, as YYYY-MM-DD."""
+    dates = []
+    for key in ("document_date", "date", "created_at"):
+        value = metadata.get(key)
+        if isinstance(value, (date, datetime)):
+            dates.append(value.isoformat()[:10])
+        elif isinstance(value, str):
+            try:
+                dates.append(date.fromisoformat(value[:10]).isoformat())
+            except ValueError:
+                pass
+    try:
+        if metadata.get("created_date") is not None:
+            dates.append(
+                datetime.fromtimestamp(float(metadata["created_date"]), timezone.utc)
+                .date()
+                .isoformat()
+            )
+    except (ValueError, TypeError, OverflowError, OSError):
+        pass
+    return dates
+
+
+def document_date(metadata: dict) -> str | None:
+    """The date shown for a citation: the most specific one available."""
+    dates = document_dates(metadata)
+    return dates[0] if dates else None
+
+
+def citation_url(document) -> str | None:
+    try:
+        return "/sources/" + str(UUID(str(document.id)))
+    except (ValueError, TypeError, AttributeError):
+        return None

blueprints/rag/source_api.py

@@ -0,0 +1,86 @@
+"""Read-only source chunks from the same authenticated shared retrieval corpus."""
+
+import asyncio
+import os
+from urllib.parse import urlsplit
+from uuid import UUID
+
+from quart import Blueprint, jsonify
+from sqlalchemy import text
+
+from blueprints.conversation.artifacts_util import (
+    obsidian_source_ref,
+    vault_relative_path,
+)
+from blueprints.users.decorators import jwt_access_required
+from .retrieval import document_date
+
+sources_blueprint = Blueprint("sources_api", __name__, url_prefix="/api/sources")
+
+
+def _load_chunk(identifier: str) -> dict | None:
+    from .logic import _get_collection_id, _get_engine
+
+    collection = _get_collection_id()
+    if not collection:
+        return None
+    with _get_engine().connect() as conn:
+        row = conn.execute(
+            text(
+                "SELECT document, cmetadata FROM langchain_pg_embedding WHERE id=:id AND collection_id=:collection"
+            ),
+            {"id": identifier, "collection": collection},
+        ).first()
+    if row is None:
+        return None
+    metadata = row[1] or {}
+    source = metadata.get("source")
+    if source not in {"obsidian", "paperless"}:
+        return None
+    result = {
+        "id": identifier,
+        "source": source,
+        "content": row[0],
+        "date": document_date(metadata),
+        "headings": [metadata[k] for k in ("h1", "h2", "h3") if metadata.get(k)],
+    }
+    if source == "obsidian":
+        path = vault_relative_path(metadata.get("filepath") or "")
+        result.update(title=metadata.get("title") or path or "Note", path=path)
+        result["original_url"] = obsidian_source_ref(path).get("url") if path else None
+    else:
+        result.update(
+            title=metadata.get("filename") or "Document",
+            document_type=metadata.get("document_type"),
+        )
+        base = os.getenv("BASE_URL", "").rstrip("/")
+        base = base if "://" in base else "http://" + base if base else ""
+        parsed = urlsplit(base)
+        document_id = metadata.get("document_id")
+        result["original_url"] = (
+            f"{base}/documents/{document_id}/details"
+            if parsed.scheme in {"http", "https"}
+            and parsed.hostname
+            and not parsed.username
+            and str(document_id).isdigit()
+            else None
+        )
+    return result
+
+
+@sources_blueprint.get("/<source_id>")
+@jwt_access_required
+async def get_source(source_id):
+    try:
+        identifier = str(UUID(source_id))
+    except ValueError:
+        return jsonify(error="source not found"), 404
+    try:
+        result = await asyncio.to_thread(_load_chunk, identifier)
+    except Exception:
+        return jsonify(error="Source storage is unavailable. Please try again."), 503
+    if result is None:
+        return jsonify(error="source not found"), 404
+    response = jsonify(result)
+    response.headers["Cache-Control"] = "no-store"
+    return response

blueprints/scheduled_messages/__init__.py

@@ -3,7 +3,9 @@ import logging
 import os
 from datetime import datetime, timezone
 
+from .validation import json_object
 from quart import Blueprint, request, jsonify
+from tortoise.expressions import Q
 
 from blueprints.users.decorators import admin_required
 from .models import ScheduledMessage, MessageChannel, MessageStatus, Recurrence
@@ -18,6 +20,8 @@ logger = logging.getLogger(__name__)
 def _serialize(msg: ScheduledMessage) -> dict:
     return {
         "id": str(msg.id),
+        "series_id": str(msg.series_id or msg.id),
+        "retryable": msg.status == MessageStatus.FAILED,
         "recipient": msg.recipient,
         "channel": msg.channel.value,
         "content": msg.content,
@@ -35,16 +39,29 @@ def _serialize(msg: ScheduledMessage) -> dict:
 @scheduled_messages_blueprint.route("/", methods=["GET"])
 @admin_required
 async def list_messages():
-    messages = await ScheduledMessage.all().order_by("-scheduled_at")
+    messages = await ScheduledMessage.filter(deleted_at__isnull=True).order_by(
+        "-scheduled_at"
+    )
     return jsonify([_serialize(m) for m in messages])
 
 
 @scheduled_messages_blueprint.route("/", methods=["POST"])
 @admin_required
 async def create_message():
-    data = await request.get_json()
-    if not data:
-        return jsonify({"error": "Invalid payload"}), 400
+    data, error = await json_object(
+        (
+            "recipient",
+            "channel",
+            "content",
+            "subject",
+            "scheduled_at",
+            "recurrence",
+            "status",
+        ),
+        nullable=("subject",),
+    )
+    if error:
+        return error
 
     recipient = (data.get("recipient") or "").strip()
     channel = data.get("channel")
@@ -112,19 +129,109 @@ async def create_message():
     return jsonify(_serialize(msg)), 201
 
 
+async def _cancel(msg):
+    """Cancel an occurrence and stop its series without rewriting delivery outcomes.
+
+    Rows whose delivery has started sending keep their status (sent, uncertain or
+    in flight); only work that has not reached a provider is cancelled.
+    """
+    from tortoise.transactions import in_transaction
+
+    from .models import JobDelivery
+
+    async with in_transaction() as db:
+        series = msg.series_id or msg.id
+        # Same lock order as delivery: series row first, so the "sending"
+        # transition cannot interleave with this check.
+        await (
+            ScheduledMessage.filter(id=series)
+            .using_db(db)
+            .update(updated_at=datetime.now(timezone.utc))
+        )
+        members = await ScheduledMessage.filter(
+            Q(id=msg.id) | Q(series_id=series),
+            status__in=[
+                MessageStatus.PENDING,
+                MessageStatus.FAILED,
+                MessageStatus.PROCESSING,
+            ],
+        ).using_db(db)
+        started = set(
+            await JobDelivery.filter(
+                kind="scheduled",
+                job_id__in=[str(m.id) for m in members],
+                status__in=["sending", "sent", "uncertain"],
+            )
+            .using_db(db)
+            .values_list("job_id", flat=True)
+        )
+        cancellable = [m.id for m in members if str(m.id) not in started]
+        if cancellable:
+            await (
+                ScheduledMessage.filter(id__in=cancellable)
+                .using_db(db)
+                .update(status=MessageStatus.CANCELLED, series_id=series)
+            )
+        # Claiming sets series_id to the row's own id, so only a different series
+        # id (or a predecessor) marks a one-off row as part of a series.
+        recurring = (
+            msg.recurrence != Recurrence.NONE
+            or msg.previous_occurrence_id
+            or (msg.series_id and msg.series_id != msg.id)
+        )
+        if msg.id not in cancellable and not recurring:
+            return jsonify(
+                {"error": "Delivery already started; it cannot be cancelled"}
+            ), 409
+        if recurring and not cancellable:
+            # Nothing pending to mark: a hidden tombstone stops future occurrences.
+            await ScheduledMessage.create(
+                series_id=series,
+                recipient=msg.recipient,
+                channel=msg.channel,
+                content=msg.content,
+                subject=msg.subject,
+                scheduled_at=msg.scheduled_at,
+                recurrence=Recurrence.NONE,
+                status=MessageStatus.CANCELLED,
+                deleted_at=datetime.now(timezone.utc),
+                created_by_id=msg.created_by_id,
+                using_db=db,
+            )
+    await msg.refresh_from_db()
+    return jsonify(_serialize(msg))
+
+
 @scheduled_messages_blueprint.route("/<msg_id>", methods=["PUT"])
 @admin_required
 async def update_message(msg_id: str):
-    msg = await ScheduledMessage.get_or_none(id=msg_id)
+    msg = await ScheduledMessage.get_or_none(id=msg_id, deleted_at__isnull=True)
     if not msg:
         return jsonify({"error": "Not found"}), 404
 
+    data, error = await json_object(
+        (
+            "recipient",
+            "channel",
+            "content",
+            "subject",
+            "scheduled_at",
+            "recurrence",
+            "status",
+        ),
+        nullable=("subject",),
+    )
+    if error:
+        return error
+
+    if data.get("status") == "cancelled":
+        return await _cancel(msg)
     if msg.status not in (MessageStatus.PENDING, MessageStatus.FAILED):
         return jsonify({"error": "Can only update pending or failed messages"}), 400
 
-    data = await request.get_json()
-    if not data:
-        return jsonify({"error": "Invalid payload"}), 400
+    blank = [f for f in ("recipient", "content") if f in data and not data[f].strip()]
+    if blank:
+        return jsonify({"error": f"{', '.join(blank)} cannot be empty"}), 400
 
     rescheduled = False
     if "recipient" in data:
@@ -137,7 +244,7 @@ async def update_message(msg_id: str):
     if "content" in data:
         msg.content = data["content"].strip()
     if "subject" in data:
-        msg.subject = data["subject"].strip() or None
+        msg.subject = (data["subject"] or "").strip() or None
     if "recurrence" in data:
         try:
             msg.recurrence = Recurrence(data["recurrence"])
@@ -156,16 +263,31 @@ async def update_message(msg_id: str):
             rescheduled = True
         except ValueError:
             return jsonify({"error": "Invalid scheduled_at format"}), 400
-    if "status" in data and data["status"] == "cancelled":
-        msg.status = MessageStatus.CANCELLED
-    elif rescheduled:
-        msg.status = MessageStatus.PENDING
+    if rescheduled and msg.status == MessageStatus.PENDING:
         msg.error_message = None
 
     if msg.channel == MessageChannel.EMAIL and not msg.subject:
         return jsonify({"error": "subject is required for email messages"}), 400
 
-    await msg.save()
+    values = {
+        field: getattr(msg, field)
+        for field in (
+            "recipient",
+            "channel",
+            "content",
+            "subject",
+            "scheduled_at",
+            "recurrence",
+            "use_agent",
+            "status",
+            "error_message",
+        )
+    }
+    changed = await ScheduledMessage.filter(
+        id=msg.id, status__in=[MessageStatus.PENDING, MessageStatus.FAILED]
+    ).update(**values)
+    if not changed:
+        return jsonify({"error": "Message was claimed; update rejected"}), 409
 
     if rescheduled and msg.status == MessageStatus.PENDING:
         from utils.cronicle import create_message_event
@@ -197,12 +319,80 @@ async def trigger_due_messages():
 @scheduled_messages_blueprint.route("/<msg_id>", methods=["DELETE"])
 @admin_required
 async def delete_message(msg_id: str):
-    msg = await ScheduledMessage.get_or_none(id=msg_id)
+    msg = await ScheduledMessage.get_or_none(id=msg_id, deleted_at__isnull=True)
     if not msg:
         return jsonify({"error": "Not found"}), 404
 
     if msg.status not in (MessageStatus.PENDING, MessageStatus.CANCELLED):
         return jsonify({"error": "Can only delete pending or cancelled messages"}), 400
 
-    await msg.delete()
+    if msg.recurrence != Recurrence.NONE or msg.series_id or msg.previous_occurrence_id:
+        # Keep a durable cancellation tombstone so recurrence repair cannot revive
+        # it, even if this occurrence was since edited to be non-recurring.
+        from tortoise.transactions import in_transaction
+
+        async with in_transaction() as db:
+            series = msg.series_id or msg.id
+            await (
+                ScheduledMessage.filter(id=series)
+                .using_db(db)
+                .update(updated_at=datetime.now(timezone.utc))
+            )
+            deleted = (
+                await ScheduledMessage.filter(
+                    id=msg.id,
+                    status__in=[MessageStatus.PENDING, MessageStatus.CANCELLED],
+                )
+                .using_db(db)
+                .update(
+                    status=MessageStatus.CANCELLED,
+                    series_id=series,
+                    deleted_at=datetime.now(timezone.utc),
+                )
+            )
+            if not deleted:
+                return jsonify({"error": "Message was claimed; deletion rejected"}), 409
+            await (
+                ScheduledMessage.filter(
+                    series_id=series,
+                    status__in=[
+                        MessageStatus.PENDING,
+                        MessageStatus.FAILED,
+                        MessageStatus.PROCESSING,
+                    ],
+                )
+                .using_db(db)
+                .update(status=MessageStatus.CANCELLED)
+            )
+    else:
+        deleted = await ScheduledMessage.filter(
+            id=msg.id, status__in=[MessageStatus.PENDING, MessageStatus.CANCELLED]
+        ).delete()
+        if not deleted:
+            return jsonify({"error": "Message was claimed; deletion rejected"}), 409
     return jsonify({"status": "deleted"})
+
+
+@scheduled_messages_blueprint.route("/<msg_id>/deliveries", methods=["GET"])
+@admin_required
+async def delivery_history(msg_id):
+    if not await ScheduledMessage.filter(id=msg_id, deleted_at__isnull=True).exists():
+        return jsonify({"error": "Not found"}), 404
+    from .delivery import history
+
+    return jsonify(await history("scheduled", msg_id))
+
+
+@scheduled_messages_blueprint.route("/<msg_id>/retry", methods=["POST"])
+@admin_required
+async def retry_message(msg_id):
+    msg = await ScheduledMessage.get_or_none(id=msg_id, deleted_at__isnull=True)
+    if not msg:
+        return jsonify({"error": "Not found"}), 404
+    if msg.status != MessageStatus.FAILED:
+        return jsonify({"error": "Only safe failed deliveries can be retried"}), 409
+    from .scheduler import process_message
+
+    status = await process_message(msg, retry=True)
+    await msg.refresh_from_db()
+    return jsonify(_serialize(msg)), 200 if status == "sent" else 409

blueprints/scheduled_messages/delivery.py

@@ -0,0 +1,171 @@
+"""DB compare-and-set claims shared by scheduled and cron delivery.
+
+The sending marker is committed BEFORE calling the provider. Any provider
+exception is ambiguous unless it explicitly guarantees no external effect.
+"""
+
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import httpx
+from tortoise.transactions import in_transaction
+
+from .models import JobAttempt, JobDelivery
+
+
+class SafeDeliveryError(Exception):
+    """Provider rejected the request before any delivery could occur."""
+
+
+def now():
+    return datetime.now(timezone.utc)
+
+
+async def claim(kind, job_id, run_id, *, retry=False):
+    delivery, _ = await JobDelivery.get_or_create(
+        kind=kind, job_id=str(job_id), run_id=run_id
+    )
+    token = uuid.uuid4()
+    async with in_transaction() as db:
+        won = (
+            await JobDelivery.filter(
+                id=delivery.id, status="failed" if retry else "pending"
+            )
+            .using_db(db)
+            .update(
+                status="processing", claim_token=token, claimed_at=now(), error=None
+            )
+        )
+        if not won:
+            return None
+        await JobAttempt.create(delivery_id=delivery.id, claim_token=token, using_db=db)
+    return await JobDelivery.get(id=delivery.id)
+
+
+async def transition(delivery, status, error=None):
+    async with in_transaction() as db:
+        won = (
+            await JobDelivery.filter(
+                id=delivery.id,
+                claim_token=delivery.claim_token,
+                status__in=["processing", "sending"],
+            )
+            .using_db(db)
+            .update(status=status, error=error, updated_at=now())
+        )
+        if won:
+            await (
+                JobAttempt.filter(claim_token=delivery.claim_token)
+                .using_db(db)
+                .update(
+                    status=status,
+                    error=error,
+                    finished_at=None if status == "sending" else now(),
+                )
+            )
+    return bool(won)
+
+
+async def expire_claims(kind, *, age=timedelta(hours=1)):
+    """Abandoned work requires reconciliation, never an automatic resend."""
+    stale = await JobDelivery.filter(
+        kind=kind, status__in=["processing", "sending"], claimed_at__lt=now() - age
+    )
+    for delivery in stale:
+        await transition(
+            delivery, "uncertain", "Worker abandoned; delivery requires reconciliation"
+        )
+
+
+async def agent_content(user, prompt, thread_id, channel):
+    from blueprints.conversation.runtime import invoke_agent_for_user
+    from utils.strip_markdown import extract_agent_text
+
+    response = await invoke_agent_for_user(user, prompt, thread_id, channel=channel)
+    messages = response.get("messages", [])
+    if not messages:
+        raise ValueError("Agent returned no messages")
+    last = messages[-1]
+    content = last.get("content") if isinstance(last, dict) else last.content
+    content = extract_agent_text(content)
+    if not content:
+        raise ValueError("Agent returned no text")
+    return content
+
+
+async def send(channel, recipient, content, subject=None):
+    from .models import MessageChannel
+
+    try:
+        if channel == MessageChannel.IMESSAGE:
+            from blueprints.imessage import send_imessage
+            from utils.strip_markdown import strip_markdown
+
+            await send_imessage(recipient, strip_markdown(content))
+        elif channel == MessageChannel.EMAIL:
+            # The shared reply helper logs provider failures and returns None.
+            # Delivery tracking requires an explicit provider acknowledgement.
+            api_key = os.getenv("MAILGUN_API_KEY")
+            domain = os.getenv("MAILGUN_DOMAIN")
+            if not api_key or not domain:
+                raise SafeDeliveryError(
+                    "MAILGUN_API_KEY or MAILGUN_DOMAIN not configured"
+                )
+            subject = subject or "(no subject)"
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    f"https://api.mailgun.net/v3/{domain}/messages",
+                    auth=("api", api_key),
+                    data={
+                        "from": f"Simba <simba@{domain}>",
+                        "to": recipient,
+                        "subject": subject
+                        if subject.startswith("Re:")
+                        else f"Re: {subject}",
+                        "text": content,
+                    },
+                    timeout=30,
+                )
+                response.raise_for_status()
+        else:
+            raise SafeDeliveryError("Unsupported channel")
+    except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as exc:
+        raise SafeDeliveryError(str(exc)) from exc
+    except httpx.HTTPStatusError as exc:
+        # Only explicit request rejection is safe. Server errors/timeouts may
+        # follow an accepted delivery and must remain uncertain.
+        if exc.response.status_code in {400, 401, 403, 404, 405, 413, 422, 429}:
+            raise SafeDeliveryError(str(exc)) from exc
+        raise
+
+
+async def history(kind, job_id):
+    deliveries = await JobDelivery.filter(kind=kind, job_id=str(job_id)).order_by(
+        "-created_at"
+    )
+    result = []
+    for d in deliveries:
+        attempts = await JobAttempt.filter(delivery_id=d.id).order_by("started_at")
+        result.append(
+            {
+                "id": str(d.id),
+                "run_id": d.run_id,
+                "status": d.status,
+                "error": d.error,
+                "retryable": d.status == "failed",
+                "attempts": [
+                    {
+                        "id": str(a.id),
+                        "status": a.status,
+                        "error": a.error,
+                        "started_at": a.started_at.isoformat(),
+                        "finished_at": a.finished_at.isoformat()
+                        if a.finished_at
+                        else None,
+                    }
+                    for a in attempts
+                ],
+            }
+        )
+    return result

blueprints/scheduled_messages/models.py

@@ -11,6 +11,8 @@ class MessageChannel(enum.Enum):
 
 class MessageStatus(enum.Enum):
     PENDING = "pending"
+    PROCESSING = "processing"
+    UNCERTAIN = "uncertain"
     SENT = "sent"
     FAILED = "failed"
     CANCELLED = "cancelled"
@@ -25,6 +27,14 @@ class Recurrence(enum.Enum):
 
 class ScheduledMessage(Model):
     id = fields.UUIDField(primary_key=True)
+    deleted_at = fields.DatetimeField(null=True)
+    series_id = fields.UUIDField(null=True)
+    previous_occurrence = fields.OneToOneField(
+        "models.ScheduledMessage",
+        null=True,
+        related_name="next_occurrence",
+        on_delete=fields.SET_NULL,
+    )
     recipient = fields.CharField(max_length=255)
     channel = fields.CharEnumField(enum_type=MessageChannel, max_length=20)
     content = fields.TextField()
@@ -46,3 +56,35 @@ class ScheduledMessage(Model):
 
     class Meta:
         table = "scheduled_messages"
+
+
+class JobDelivery(Model):
+    """Durable run identity; never reclaim ambiguous external deliveries."""
+
+    id = fields.UUIDField(primary_key=True)
+    kind = fields.CharField(max_length=20)
+    job_id = fields.CharField(max_length=36)
+    run_id = fields.CharField(max_length=255)
+    status = fields.CharField(max_length=20, default="pending")
+    claim_token = fields.UUIDField(null=True)
+    claimed_at = fields.DatetimeField(null=True)
+    error = fields.TextField(null=True)
+    created_at = fields.DatetimeField(auto_now_add=True)
+    updated_at = fields.DatetimeField(auto_now=True)
+
+    class Meta:
+        table = "job_deliveries"
+        unique_together = (("kind", "job_id", "run_id"),)
+
+
+class JobAttempt(Model):
+    id = fields.UUIDField(primary_key=True)
+    delivery = fields.ForeignKeyField("models.JobDelivery", related_name="attempts")
+    claim_token = fields.UUIDField(unique=True)
+    status = fields.CharField(max_length=20, default="processing")
+    error = fields.TextField(null=True)
+    started_at = fields.DatetimeField(auto_now_add=True)
+    finished_at = fields.DatetimeField(null=True)
+
+    class Meta:
+        table = "job_attempts"

blueprints/scheduled_messages/scheduler.py

@@ -1,13 +1,13 @@
 import logging
-import uuid
 from datetime import datetime, timezone
-from dateutil.relativedelta import relativedelta
-from quart import current_app
 
-from .models import ScheduledMessage, MessageChannel, MessageStatus, Recurrence
+from dateutil.relativedelta import relativedelta
+from tortoise.transactions import in_transaction
+
+from . import delivery as transport
+from .models import ScheduledMessage, MessageStatus, Recurrence, JobDelivery
 
 logger = logging.getLogger(__name__)
-
 RECURRENCE_DELTAS = {
     Recurrence.DAILY: relativedelta(days=1),
     Recurrence.WEEKLY: relativedelta(weeks=1),
@@ -15,88 +15,178 @@ RECURRENCE_DELTAS = {
 }
 
 
-async def _run_agent(prompt: str, thread_id: str) -> str:
-    response = await current_app.main_agent.ainvoke(
-        {"messages": [{"role": "user", "content": prompt}]},
-        config={"configurable": {"thread_id": thread_id}},
-    )
-    return response.get("messages", [])[-1].content
+def run_id(msg):
+    # Each recurring occurrence has its own row; edits keep the same identity.
+    return str(msg.id)
 
 
-async def _schedule_next_occurrence(msg: ScheduledMessage):
+async def _schedule_next_occurrence(msg, db=None):
+    if db is None:
+        async with in_transaction() as connection:
+            return await _schedule_next_occurrence(msg, connection)
     delta = RECURRENCE_DELTAS.get(msg.recurrence)
     if not delta:
         return
-
+    series = msg.series_id or msg.id
+    # Serialize series creation/cancellation on the original occurrence.
+    await (
+        ScheduledMessage.filter(id=series)
+        .using_db(db)
+        .update(updated_at=transport.now())
+    )
+    if (
+        await ScheduledMessage.filter(series_id=series, status=MessageStatus.CANCELLED)
+        .using_db(db)
+        .exists()
+    ):
+        return
     next_at = msg.scheduled_at + delta
     now = datetime.now(timezone.utc)
     while next_at <= now:
         next_at += delta
+    await ScheduledMessage.get_or_create(
+        previous_occurrence_id=msg.id,
+        defaults=dict(
+            series_id=series,
+            recipient=msg.recipient,
+            channel=msg.channel,
+            content=msg.content,
+            subject=msg.subject,
+            scheduled_at=next_at,
+            recurrence=msg.recurrence,
+            use_agent=msg.use_agent,
+            created_by_id=msg.created_by_id,
+        ),
+        using_db=db,
+    )
 
-    await ScheduledMessage.create(
-        recipient=msg.recipient,
-        channel=msg.channel,
-        content=msg.content,
-        subject=msg.subject,
-        scheduled_at=next_at,
-        recurrence=msg.recurrence,
-        use_agent=msg.use_agent,
-        created_by_id=msg.created_by_id,
-    )
-    logger.info(
-        f"Scheduled next {msg.recurrence.value} occurrence for {msg.id} at {next_at.isoformat()}"
-    )
+
+async def process_message(msg, *, retry=False):
+    async with in_transaction() as db:
+        # Always lock the series before an occurrence, matching cancellation.
+        await (
+            ScheduledMessage.filter(id=msg.series_id or msg.id)
+            .using_db(db)
+            .update(updated_at=transport.now())
+        )
+        eligible = ScheduledMessage.filter(
+            id=msg.id, status=MessageStatus.FAILED if retry else MessageStatus.PENDING
+        )
+        if not retry:
+            eligible = eligible.filter(scheduled_at__lte=transport.now())
+        won = await eligible.using_db(db).update(
+            status=MessageStatus.PROCESSING, series_id=msg.series_id or msg.id
+        )
+        if not won:
+            return "skipped"
+        msg = await ScheduledMessage.get(id=msg.id).using_db(db)
+        claimed = await transport.claim("scheduled", msg.id, run_id(msg), retry=retry)
+        if not claimed:
+            # Preserve the authoritative delivery state if the job was rescheduled
+            # onto an occurrence already delivered.
+            d = await JobDelivery.get(
+                kind="scheduled", job_id=str(msg.id), run_id=run_id(msg)
+            )
+            status = (
+                MessageStatus(d.status)
+                if d.status in {s.value for s in MessageStatus}
+                else MessageStatus.UNCERTAIN
+            )
+            await ScheduledMessage.filter(id=msg.id).using_db(db).update(status=status)
+            return "skipped"
+    sending = False
+    try:
+        content = msg.content
+        if msg.use_agent:
+            user = await msg.created_by
+            content = await transport.agent_content(
+                user, content, f"scheduled-{claimed.id}", "scheduled"
+            )
+        # A cancellation which wins before this transaction prevents sending.
+        async with in_transaction() as db:
+            await (
+                ScheduledMessage.filter(id=msg.series_id or msg.id)
+                .using_db(db)
+                .update(updated_at=transport.now())
+            )
+            active = (
+                await ScheduledMessage.filter(
+                    id=msg.id, status=MessageStatus.PROCESSING
+                )
+                .using_db(db)
+                .update(updated_at=transport.now())
+            )
+            if not active:
+                await transport.transition(claimed, "cancelled")
+                return "cancelled"
+            if not await transport.transition(claimed, "sending"):
+                return "skipped"
+        sending = True
+        await transport.send(msg.channel, msg.recipient, content, msg.subject)
+        async with in_transaction() as db:
+            await (
+                ScheduledMessage.filter(id=msg.series_id or msg.id)
+                .using_db(db)
+                .update(updated_at=transport.now())
+            )
+            await transport.transition(claimed, "sent")
+            active = (
+                await ScheduledMessage.filter(
+                    id=msg.id, status=MessageStatus.PROCESSING
+                )
+                .using_db(db)
+                .update(status=MessageStatus.SENT, error_message=None)
+            )
+            if active:
+                await _schedule_next_occurrence(msg, db)
+        return "sent"
+    except Exception as exc:
+        status = (
+            "uncertain"
+            if sending and not isinstance(exc, transport.SafeDeliveryError)
+            else "failed"
+        )
+        await transport.transition(claimed, status, str(exc))
+        await ScheduledMessage.filter(
+            id=msg.id, status=MessageStatus.PROCESSING
+        ).update(status=MessageStatus(status), error_message=str(exc))
+        logger.exception("Scheduled delivery %s ended as %s", msg.id, status)
+        return status
 
 
 async def process_due_messages() -> dict:
-    """Process all currently due pending messages. Returns a result summary."""
-    now = datetime.now(timezone.utc)
-    due = await ScheduledMessage.filter(
-        status=MessageStatus.PENDING,
-        scheduled_at__lte=now,
-    ).all()
-
-    sent, failed = 0, 0
-    for msg in due:
-        try:
-            send_content = msg.content
-
-            if msg.use_agent:
-                send_content = await _run_agent(
-                    msg.content, f"scheduled-{uuid.uuid4()}"
-                )
-
-            if msg.channel == MessageChannel.IMESSAGE:
-                from blueprints.imessage import send_imessage
-                from utils.strip_markdown import strip_markdown
-
-                await send_imessage(msg.recipient, strip_markdown(send_content))
-
-            elif msg.channel == MessageChannel.EMAIL:
-                from blueprints.email import send_email_reply
-
-                await send_email_reply(
-                    to=msg.recipient,
-                    subject=msg.subject or "(no subject)",
-                    body=send_content,
-                )
-
-            msg.status = MessageStatus.SENT
-            msg.error_message = None
-            await msg.save()
-            logger.info(
-                f"Sent scheduled {msg.channel.value} message {msg.id} to {msg.recipient}"
-            )
-            sent += 1
-
-            if msg.recurrence != Recurrence.NONE:
+    await transport.expire_claims("scheduled")
+    uncertain = await JobDelivery.filter(kind="scheduled", status="uncertain")
+    for d in uncertain:
+        await ScheduledMessage.filter(
+            id=d.job_id, status=MessageStatus.PROCESSING
+        ).update(status=MessageStatus.UNCERTAIN, error_message=d.error)
+    # Recover recurrence creation if an earlier successful send was committed.
+    # Only rows sent through delivery claims qualify: rows sent before migration
+    # 20 have no successor links, and repairing them would fork each historical
+    # occurrence into its own new series.
+    orphans = await ScheduledMessage.filter(
+        status=MessageStatus.SENT,
+        deleted_at__isnull=True,
+        next_occurrence__id__isnull=True,
+    ).exclude(recurrence=Recurrence.NONE)
+    if orphans:
+        claimed_sends = set(
+            await JobDelivery.filter(
+                kind="scheduled",
+                status="sent",
+                job_id__in=[str(m.id) for m in orphans],
+            ).values_list("job_id", flat=True)
+        )
+        for msg in orphans:
+            if str(msg.id) in claimed_sends:
                 await _schedule_next_occurrence(msg)
-
-        except Exception as e:
-            msg.status = MessageStatus.FAILED
-            msg.error_message = str(e)
-            await msg.save()
-            logger.error(f"Failed to send scheduled message {msg.id}: {e}")
-            failed += 1
-
-    return {"sent": sent, "failed": failed}
+    due = await ScheduledMessage.filter(
+        status=MessageStatus.PENDING, scheduled_at__lte=transport.now()
+    )
+    counts = {"sent": 0, "failed": 0, "uncertain": 0}
+    for msg in due:
+        result = await process_message(msg)
+        if result in counts:
+            counts[result] += 1
+    return counts

blueprints/scheduled_messages/validation.py

@@ -0,0 +1,25 @@
+"""Request-body checks shared by the scheduled message and cron endpoints."""
+
+from quart import jsonify, request
+
+
+async def json_object(text_fields: tuple[str, ...], nullable: tuple[str, ...] = ()):
+    """Return (data, None) for a non-empty object whose text fields are strings.
+
+    Fields in `nullable` may also be an explicit null. Otherwise return
+    (None, error_response) so malformed bodies get a 400 instead of failing on
+    `.get`/`.strip` with a 500.
+    """
+    data = await request.get_json(silent=True)
+    if not isinstance(data, dict) or not data:
+        return None, (jsonify({"error": "Invalid payload"}), 400)
+    wrong = [
+        name
+        for name in text_fields
+        if name in data
+        and not isinstance(data[name], str)
+        and not (data[name] is None and name in nullable)
+    ]
+    if wrong:
+        return None, (jsonify({"error": f"{', '.join(wrong)} must be text"}), 400)
+    return data, None

blueprints/whatsapp/__init__.py

@@ -9,11 +9,11 @@ from twilio.twiml.messaging_response import MessagingResponse
 
 from blueprints.users.models import User
 from blueprints.conversation.logic import (
-    get_agent_for,
     get_conversation_for_channel,
     add_message_to_conversation,
     persist_agent_tool_messages,
 )
+from blueprints.conversation.runtime import invoke_agent_for_user
 from utils.strip_markdown import extract_agent_text
 
 whatsapp_blueprint = Blueprint("whatsapp_api", __name__, url_prefix="/api/whatsapp")
@@ -177,17 +177,11 @@ async def webhook():
     )
 
     try:
-        agent_config = {
-            "configurable": {
-                "user_id": str(user.id),
-                "is_admin": user.is_admin(),
-                "thread_id": str(conversation.id),
-            }
-        }
-        logger.info(f"Invoking LangChain agent for conversation {conversation.id}")
-        response = await get_agent_for(user).ainvoke(
-            {"messages": [{"role": "user", "content": body}]},
-            config=agent_config,
+        response = await invoke_agent_for_user(
+            user,
+            body,
+            str(conversation.id),
+            channel="whatsapp",
         )
         response_text = extract_agent_text(response.get("messages", [])[-1].content)
         await persist_agent_tool_messages(

blueprints/ynab/sync.py

@@ -14,6 +14,7 @@ import asyncio
 import logging
 import os
 import time
+from contextvars import ContextVar
 from dataclasses import dataclass
 from datetime import date, datetime, timedelta, timezone
 from decimal import Decimal
@@ -46,6 +47,9 @@ FAILURE_BACKOFF_SECONDS = 60
 LOOKUP_CHUNK = 2000
 
 _sync_lock = asyncio.Lock()
+tool_data_freshness: ContextVar[dict | None] = ContextVar(
+    "ynab_tool_data_freshness", default=None
+)
 _last_failure = float("-inf")
 
 
@@ -616,6 +620,7 @@ async def ensure_fresh(max_age_seconds: int = 120) -> YnabSyncState | None:
     ``FAILURE_BACKOFF_SECONDS``. Returns None only if the mirror has never
     been populated.
     """
+    tool_data_freshness.set(None)
     state = await _current_state()
     if not _is_fresh(state, max_age_seconds) and not _backing_off():
         try:
@@ -625,6 +630,14 @@ async def ensure_fresh(max_age_seconds: int = 120) -> YnabSyncState | None:
         state = await _current_state()
     if state is None or state.server_knowledge is None:
         return None
+    stamp = _norm(state.last_synced_at) if state.last_synced_at else None
+    tool_data_freshness.set(
+        {
+            "system": "ynab",
+            "as_of": stamp.isoformat() if stamp else None,
+            "stale": not _is_fresh(state, max_age_seconds),
+        }
+    )
     return state
 
 

blueprints/ynab/tools.py

@@ -21,7 +21,7 @@ from blueprints.conversation.artifacts_util import (
     make_artifact,
 )
 
-from .sync import ensure_fresh
+from .sync import ensure_fresh, tool_data_freshness
 
 logger = logging.getLogger(__name__)
 
@@ -64,8 +64,32 @@ def ynab_tool_errors(fn):
 
     @functools.wraps(fn)
     async def wrapper(*args, **kwargs):
+        tool_data_freshness.set(None)
         try:
-            return await fn(*args, **kwargs)
+            result = await fn(*args, **kwargs)
+            freshness = tool_data_freshness.get()
+            if not freshness:
+                return result
+            warning = (
+                (
+                    f"\nData last synchronized: {freshness['as_of'] or 'unknown'}. "
+                    "WARNING: cached budget data is stale; disclose this when answering."
+                )
+                if freshness["stale"]
+                else ""
+            )
+            if isinstance(result, tuple) and len(result) == 2:
+                content, artifact = result
+                if isinstance(artifact, dict):
+                    artifact = {
+                        **artifact,
+                        "source_ref": {
+                            **(artifact.get("source_ref") or {}),
+                            **freshness,
+                        },
+                    }
+                return str(content) + warning, artifact
+            return str(result) + warning
         except Exception as e:
             logger.exception(f"{fn.__name__} failed")
             return f"YNAB error in {fn.__name__}: {type(e).__name__}: {e}"

config/db.py

@@ -14,6 +14,7 @@ TORTOISE_CONFIG = {
         "models": {
             "models": [
                 "blueprints.conversation.models",
+                "blueprints.conversation.run_models",
                 "blueprints.users.models",
                 "blueprints.email.models",
                 "blueprints.scheduled_messages.models",

config/validation.py

@@ -0,0 +1,269 @@
+"""Explicit startup validation; importing this module never requires secrets.
+
+Errors name settings, never their values. This is intentionally independent of
+application imports, database access, and optional integration clients.
+"""
+
+import base64
+import binascii
+import math
+import posixpath
+import re
+from collections.abc import Mapping
+from urllib.parse import unquote, urlsplit
+
+
+class ConfigurationError(ValueError):
+    """One or more configuration settings are unsafe or malformed."""
+
+
+def validate_configuration(env: Mapping[str, str], *, production: bool = True) -> None:
+    errors: list[str] = []
+
+    def value(name: str) -> str:
+        return env.get(name, "").strip()
+
+    def problem(name: str, reason: str) -> None:
+        errors.append(f"{name}: {reason}")
+
+    def placeholder(text: str) -> bool:
+        text = text.lower()
+        return text in {"secret_key", "changeme", "password", "raggr_dev_password"} or (
+            text.startswith(("your-", "replace-", "replace_", "example-"))
+        )
+
+    def secret(name: str, *, required: bool = False, minimum: int = 1) -> None:
+        text = value(name)
+        if not text:
+            if required:
+                problem(name, "required for production startup")
+        elif production and (placeholder(text) or len(text) < minimum):
+            problem(
+                name, f"use a non-placeholder secret of at least {minimum} characters"
+            )
+
+    def url(name: str, *, https: bool = False) -> None:
+        text = value(name)
+        if not text:
+            return
+        try:
+            parsed = urlsplit(text)
+            valid = (
+                parsed.scheme in ({"https"} if https else {"http", "https"})
+                and parsed.hostname
+                and not parsed.username
+                and not parsed.password
+                and not parsed.fragment
+                and not any(c.isspace() for c in text)
+            )
+            # Accessing port validates malformed/out-of-range ports too.
+            if parsed.port == 0:
+                valid = False
+        except ValueError:
+            valid = False
+        if not valid:
+            problem(
+                name,
+                "must be a valid HTTPS URL" if https else "must be a valid HTTP(S) URL",
+            )
+
+    secret("JWT_SECRET_KEY", required=production, minimum=32)
+    secret("OPENAI_API_KEY", required=production)
+    secret("FERNET_KEY", required=production)
+    if value("FERNET_KEY"):
+        try:
+            if not re.fullmatch(r"[A-Za-z0-9_-]{43}=", value("FERNET_KEY")):
+                raise ValueError
+            decoded = base64.b64decode(
+                value("FERNET_KEY"), altchars=b"-_", validate=True
+            )
+            if len(decoded) != 32:
+                raise ValueError
+        except (ValueError, binascii.Error):
+            problem("FERNET_KEY", "must encode exactly 32 bytes in URL-safe base64")
+
+    database = value("DATABASE_URL")
+    if production or database:
+        try:
+            parsed = urlsplit(database)
+            valid = (
+                parsed.scheme == "postgres"
+                and parsed.hostname
+                and parsed.username
+                and parsed.password
+                and parsed.path not in {"", "/"}
+                and not parsed.fragment
+                and not any(c.isspace() for c in database)
+            )
+            if parsed.port == 0:
+                valid = False
+            if production and placeholder(unquote(parsed.password or "")):
+                valid = False
+        except ValueError:
+            valid = False
+        if not valid:
+            problem(
+                "DATABASE_URL",
+                "use postgres:// with host, user, non-placeholder password and database",
+            )
+
+    integers = {
+        "CONTEXT_WINDOW_TOKENS": 32768,
+        "PROMPT_BUDGET_TOKENS": 16000,
+        "RESPONSE_RESERVE_TOKENS": 4096,
+        "YNAB_SYNC_INTERVAL": 900,
+        "OBSIDIAN_SYNC_INTERVAL": 60,
+        "EMAIL_RATE_LIMIT_MAX": 5,
+        "EMAIL_RATE_LIMIT_WINDOW": 300,
+        "WHATSAPP_RATE_LIMIT_MAX": 10,
+        "WHATSAPP_RATE_LIMIT_WINDOW": 60,
+        "IMESSAGE_RATE_LIMIT_MAX": 10,
+        "IMESSAGE_RATE_LIMIT_WINDOW": 60,
+    }
+    numbers: dict[str, int] = {}
+    for name, default in integers.items():
+        try:
+            number = int(env.get(name, str(default)))
+            if number <= 0:
+                raise ValueError
+            numbers[name] = number
+        except ValueError:
+            problem(name, "must be a positive integer")
+    if all(
+        name in numbers
+        for name in (
+            "CONTEXT_WINDOW_TOKENS",
+            "PROMPT_BUDGET_TOKENS",
+            "RESPONSE_RESERVE_TOKENS",
+        )
+    ):
+        if (
+            numbers["PROMPT_BUDGET_TOKENS"] + numbers["RESPONSE_RESERVE_TOKENS"]
+            > numbers["CONTEXT_WINDOW_TOKENS"]
+        ):
+            problem(
+                "PROMPT_BUDGET_TOKENS",
+                "prompt budget plus response reserve exceeds context window",
+            )
+    for name, default in {
+        "TTFT_TIMEOUT_SECONDS": "30",
+        "LLAMA_TIMEOUT_SECONDS": "120",
+    }.items():
+        try:
+            number = float(env.get(name, default))
+            if not math.isfinite(number) or number <= 0:
+                raise ValueError
+        except ValueError:
+            problem(name, "must be a finite positive number")
+
+    for name in (
+        "OIDC_USE_DISCOVERY",
+        "OBSIDIAN_CONTINUOUS_SYNC",
+        "TWILIO_SIGNATURE_VALIDATION",
+        "MAILGUN_SIGNATURE_VALIDATION",
+        "SENDBLUE_SIGNATURE_VALIDATION",
+    ):
+        if name in env and env[name] not in {"true", "false"}:
+            problem(name, "must be true or false")
+    for name in (
+        "LLAMA_SERVER_URL",
+        "EMBEDDING_SERVER_URL",
+        "MEALIE_BASE_URL",
+        "HOMEBOX_URL",
+        "BOOKSTACK_BASE_URL",
+        "S3_ENDPOINT_URL",
+        "SIMBAKIT_URL",
+        "CRONICLE_URL",
+        "OLLAMA_URL",
+        "OLLAMA_HOST",
+        "TWILIO_WEBHOOK_URL",
+    ):
+        url(name)
+    for name in ("LLAMA_MODEL_NAME", "EMBEDDING_MODEL_NAME"):
+        if name in env and not value(name):
+            problem(name, "must not be empty when supplied")
+
+    # OIDC is optional; a partially configured login must not pass validation.
+    oidc = ("OIDC_ISSUER", "OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "OIDC_REDIRECT_URI")
+    if any(value(name) for name in oidc):
+        for name in oidc:
+            if not value(name):
+                problem(name, "required when OIDC is configured")
+        secret("OIDC_CLIENT_SECRET")
+        url("OIDC_ISSUER", https=production)
+        url("OIDC_REDIRECT_URI", https=production)
+        if env.get("OIDC_USE_DISCOVERY", "true") == "false":
+            for name in (
+                "OIDC_AUTHORIZATION_ENDPOINT",
+                "OIDC_TOKEN_ENDPOINT",
+                "OIDC_USERINFO_ENDPOINT",
+                "OIDC_JWKS_URI",
+            ):
+                if not value(name):
+                    problem(name, "required when OIDC discovery is disabled")
+                url(name, https=production)
+
+    for trigger, signing_key, flag in (
+        ("TWILIO_ACCOUNT_SID", "TWILIO_AUTH_TOKEN", "TWILIO_SIGNATURE_VALIDATION"),
+        (
+            "MAILGUN_DOMAIN",
+            "MAILGUN_WEBHOOK_SIGNING_KEY",
+            "MAILGUN_SIGNATURE_VALIDATION",
+        ),
+        (
+            "SENDBLUE_API_KEY",
+            "SENDBLUE_WEBHOOK_SECRET",
+            "SENDBLUE_SIGNATURE_VALIDATION",
+        ),
+    ):
+        if production:
+            if value(trigger) or value(signing_key):
+                secret(signing_key, required=True)
+            # Routes are registered even when outbound integrations are unused.
+            # Missing keys fail closed; disabling verification would bypass that.
+            if env.get(flag, "true") != "true":
+                problem(flag, "signature validation must remain enabled in production")
+    for name in ("METRICS_TOKEN", "CRON_SECRET", "EMAIL_HMAC_SECRET"):
+        secret(name, minimum=32)
+    if env.get("OBSIDIAN_SYNC_MODE", "bidirectional") not in {
+        "bidirectional",
+        "pull-only",
+        "mirror-remote",
+    }:
+        problem(
+            "OBSIDIAN_SYNC_MODE", "must be bidirectional, pull-only or mirror-remote"
+        )
+    if env.get("OBSIDIAN_CONTINUOUS_SYNC") == "true":
+        for name in ("OBSIDIAN_EMAIL", "OBSIDIAN_PASSWORD", "OBSIDIAN_VAULT_ID"):
+            if not value(name):
+                problem(name, "required when continuous sync is enabled")
+
+    paths = {
+        "CHECKPOINT_DB_PATH": env.get(
+            "CHECKPOINT_DB_PATH", "/app/checkpoints/checkpoints.db"
+        ),
+        "CONVERSATION_SUMMARY_DB_PATH": env.get(
+            "CONVERSATION_SUMMARY_DB_PATH",
+            env.get("CHECKPOINT_DB_PATH", "/app/checkpoints/checkpoints.db")
+            + ".summaries.db",
+        ),
+    }
+    for name, path in paths.items():
+        if (
+            not path
+            or path == ":memory:"
+            or "\x00" in path
+            or (production and not path.startswith("/"))
+            or path.endswith("/")
+        ):
+            problem(name, "must be a persistent file path (absolute in production)")
+    if posixpath.normpath(paths["CHECKPOINT_DB_PATH"]) == posixpath.normpath(
+        paths["CONVERSATION_SUMMARY_DB_PATH"]
+    ):
+        problem(
+            "CONVERSATION_SUMMARY_DB_PATH", "must differ from checkpoint database path"
+        )
+    if errors:
+        raise ConfigurationError(
+            "Invalid configuration:\n" + "\n".join(f"- {error}" for error in errors)
+        )

docker-compose.yml

@@ -1,27 +1,48 @@
-version: "3.8"
-
 x-raggr-common: &raggr-common
   build:
     context: .
     dockerfile: Dockerfile
   image: torrtle/simbarag:latest
   environment:
+    - METRICS_TOKEN=${METRICS_TOKEN:-}
+    - TTFT_TIMEOUT_SECONDS=${TTFT_TIMEOUT_SECONDS:-30}
+    - LLAMA_TIMEOUT_SECONDS=${LLAMA_TIMEOUT_SECONDS:-120}
+    - CONTEXT_WINDOW_TOKENS=${CONTEXT_WINDOW_TOKENS:-32768}
+    - PROMPT_BUDGET_TOKENS=${PROMPT_BUDGET_TOKENS:-16000}
+    - RESPONSE_RESERVE_TOKENS=${RESPONSE_RESERVE_TOKENS:-4096}
+    - CHECKPOINT_DB_PATH=${CHECKPOINT_DB_PATH:-/app/checkpoints/checkpoints.db}
+    - CHECKPOINT_KEEP_PER_THREAD=${CHECKPOINT_KEEP_PER_THREAD:-20}
+    - CONVERSATION_SUMMARY_DB_PATH=${CONVERSATION_SUMMARY_DB_PATH:-${CHECKPOINT_DB_PATH:-/app/checkpoints/checkpoints.db}.summaries.db}
+    - CHAT_RUN_LEASE_SECONDS=${CHAT_RUN_LEASE_SECONDS:-30}
+    - CHAT_RUN_MAX_SECONDS=${CHAT_RUN_MAX_SECONDS:-1800}
+    - CHAT_RUN_DRAIN_SECONDS=${CHAT_RUN_DRAIN_SECONDS:-1800}
+    - OBSIDIAN_SYNC_INTERVAL=${OBSIDIAN_SYNC_INTERVAL:-60}
+    - OIDC_AUTHORIZATION_ENDPOINT=${OIDC_AUTHORIZATION_ENDPOINT:-}
+    - OIDC_TOKEN_ENDPOINT=${OIDC_TOKEN_ENDPOINT:-}
+    - OIDC_USERINFO_ENDPOINT=${OIDC_USERINFO_ENDPOINT:-}
+    - OIDC_JWKS_URI=${OIDC_JWKS_URI:-}
+    - EMAIL_HMAC_SECRET=${EMAIL_HMAC_SECRET:-}
+    - WHATSAPP_RATE_LIMIT_MAX=${WHATSAPP_RATE_LIMIT_MAX:-10}
+    - WHATSAPP_RATE_LIMIT_WINDOW=${WHATSAPP_RATE_LIMIT_WINDOW:-60}
+    - IMESSAGE_RATE_LIMIT_MAX=${IMESSAGE_RATE_LIMIT_MAX:-10}
+    - IMESSAGE_RATE_LIMIT_WINDOW=${IMESSAGE_RATE_LIMIT_WINDOW:-60}
     - PAPERLESS_TOKEN=${PAPERLESS_TOKEN}
     - BASE_URL=${BASE_URL}
     - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434}
     - OPENAI_API_KEY=${OPENAI_API_KEY}
     - EMBEDDING_SERVER_URL=${EMBEDDING_SERVER_URL}
-    - EMBEDDING_MODEL_NAME=${EMBEDDING_MODEL_NAME}
+    - EMBEDDING_MODEL_NAME=${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
     - JWT_SECRET_KEY=${JWT_SECRET_KEY}
     - LLAMA_SERVER_URL=${LLAMA_SERVER_URL}
-    - LLAMA_MODEL_NAME=${LLAMA_MODEL_NAME}
+    - LLAMA_MODEL_NAME=${LLAMA_MODEL_NAME:-llama-3.1-8b-instruct}
+    - TOOL_ROUTING_MODE=${TOOL_ROUTING_MODE:-static}
     - OIDC_ISSUER=${OIDC_ISSUER}
     - OIDC_CLIENT_ID=${OIDC_CLIENT_ID}
     - OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET}
     - OIDC_REDIRECT_URI=${OIDC_REDIRECT_URI}
     - OIDC_USE_DISCOVERY=${OIDC_USE_DISCOVERY:-true}
     - DATABASE_URL=${DATABASE_URL:-postgres://raggr:changeme@postgres:5432/raggr}
-    - TAVILY_API_KEY=${TAVILIY_API_KEY}
+    - TAVILY_API_KEY=${TAVILY_API_KEY:-${TAVILIY_API_KEY:-}}
     - YNAB_ACCESS_TOKEN=${YNAB_ACCESS_TOKEN}
     - YNAB_BUDGET_ID=${YNAB_BUDGET_ID}
     - YNAB_SYNC_INTERVAL=${YNAB_SYNC_INTERVAL:-90}
@@ -40,6 +61,7 @@ x-raggr-common: &raggr-common
     - OBSIDIAN_E2E_PASSWORD=${OBSIDIAN_E2E_PASSWORD}
     - OBSIDIAN_DEVICE_NAME=${OBSIDIAN_DEVICE_NAME}
     - OBSIDIAN_CONTINUOUS_SYNC=${OBSIDIAN_CONTINUOUS_SYNC:-false}
+    - OBSIDIAN_SYNC_MODE=${OBSIDIAN_SYNC_MODE:-bidirectional}
     - OBSIDIAN_VAULT_PATH=${OBSIDIAN_VAULT_PATH:-/app/data/obsidian}
     - OBSIDIAN_VAULT_NAME=${OBSIDIAN_VAULT_NAME:-}
     - S3_ENDPOINT_URL=${S3_ENDPOINT_URL}

docs/chat-runs.md

@@ -0,0 +1,54 @@
+# Reconnectable chat runs
+
+A generation belongs to a durable, authenticated run rather than an HTTP connection. Closing a tab, navigating away, or losing the network detaches its subscriber. Explicit Stop sends cancellation to the run. Generation calls the authoritative `runtime.prepare_agent_turn` and `runtime.stream_prepared` under its own application context, without retaining the request context.
+
+## Wiring
+
+- `blueprints.conversation.run_models` is registered in `config/db.py`; the tables come from migration `20_20261012231017_application_foundation.py`.
+- `app.py` registers `runs_blueprint`, calls `expire_runs()` after ORM start-up (only already-expired leases; another deployment may still own live runs) and `drain_runs(app)` right after the serving lifecycle's `yield`, while the agent and checkpointer are still open.
+- `python app.py` serves through Hypercorn with `shutdown_timeout = CHAT_RUN_DRAIN_SECONDS + 30`, so lifespan shutdown outlasts the drain. `scripts/deploy.sh` gives the old container `CHAT_RUN_DRAIN_SECONDS + 60` (override with `DEPLOY_STOP_SECONDS`).
+- Every writer to a conversation checkpoint is serialized: `/runs` and `/stream-query` through `ChatRun.active_key` (cross-process), `/query` returns 409 while a run is active, and all channels share `runtime.thread_lock` (in-process).
+- Streamed deltas are coalesced (`FLUSH_SECONDS`, 150 ms) before they are written, and same-process subscribers are woken on each write; other processes fall back to polling every `POLL_SECONDS`.
+
+## API
+
+All endpoints require the existing JWT authentication. Run creation, discovery, replay and cancellation check both owner and conversation. Attachment keys must be under the authorized conversation's storage prefix.
+
+- `POST /api/conversation/runs`: `{conversation_id, query, request_id, image_key?, file_key?}`. Returns a run (`201` new, `200` idempotent replay). Clients must retain the same request ID for ambiguous retries. A reused ID with different input returns 409; another active run in the conversation returns a clear 409 before saving another user message or starting generation. Legacy callers may omit request ID and receive a fresh one.
+- `GET /api/conversation/{conversation_id}/runs`: `{run: ... | null}`. Returns the latest run, including terminal status, so loading a conversation can discover active generation or an interrupted attempt. Includes original query and image key for the brief interval before the user message is saved.
+- `GET /api/conversation/{conversation_id}/runs/{run_id}/events?cursor=N`: ordered SSE replay followed by live database polling. Only events with sequence greater than N are returned. `Last-Event-ID` is also supported when no query cursor is supplied. Negative, noninteger and ahead-of-log cursors return 400. Event `id:` is the durable sequence; event payload types remain `thinking`, `thinking_delta`, `tool_start`, `tool_end`, `content`, `response`, `error`, and terminal `[DONE]`. Comment heartbeats do not advance the cursor.
+- `POST /api/conversation/{conversation_id}/runs/{run_id}/cancel`: requests cancellation; it is idempotent. The database flag reaches the owning process through its heartbeat, normally within one second. Completion may win a race with cancellation. Keep reading until the terminal event.
+- `POST /api/conversation/stream-query`: compatibility adapter creating a run and attaching at cursor zero. Existing event types and `[DONE]` remain; `X-Chat-Run-ID` identifies the run. A disconnect detaches this endpoint too.
+
+Run statuses are `running`, `completed`, `failed`, `cancelled`, and `interrupted`. Terminal errors precede `[DONE]`. A model stall retains its existing `stalled: true` event and metrics. Transient subscriber failures are recoverable and are distinct from generation failure.
+
+## Storage and lifecycle
+
+`chat_runs` stores the authenticated owner, input, idempotency key, claim token, lease, cancellation flag, partial text and persistence markers. Its nullable unique `active_key` holds a conversation until terminalization. A composite uniqueness constraint makes duplicate creation safe across processes. `chat_run_events` has unique `(run_id, sequence)` ordering. Database transactions lock the run before appending events, finalizing or saving messages; answer and user persistence markers are committed with their messages. Tool messages/artifacts and the existing streaming metrics/timeouts stay on the established streaming path.
+
+PostgreSQL is the deployment store; real file-backed SQLite exercises the same ORM lifecycle in tests. Subscribers can land on either deployment process because replay reads shared storage. The owning process renews its lease, checks cross-process cancellation, and cancels itself if renewal fails. Expired runs save available partial text once, release admission and emit an interrupted error. A process restart **does not restart generation**. Prior persisted events and partial answers remain available; the user starts a new turn explicitly. Lease expiry assumes failed renewal promptly stops the worker; it is not fencing for independent checkpoint writers.
+
+Defaults, read from environment at module load:
+
+| Setting | Default | Purpose |
+| --- | --- | --- |
+| `CHAT_RUN_LEASE_SECONDS` | 30 | Abandoned-worker lease; minimum 10 seconds |
+| `CHAT_RUN_MAX_SECONDS` | 1800 | Entire generation, including preparation and tools |
+| `CHAT_RUN_DRAIN_SECONDS` | 1800 | Wait for tracked generations before interrupting on shutdown |
+| `DEPLOY_STOP_SECONDS` | drain + 60 | Deployment container stop grace |
+| `TTFT_TIMEOUT_SECONDS` | existing 30 | Existing per-model silent-output timeout |
+
+Subscribers have finite lifetimes. The browser retries with its last successfully handled cursor, bounds retries after consecutive transport failures, and bounds silent reads. A fresh conversation load rebuilds the active turn from cursor zero after excluding saved tool/assistant rows from that turn, avoiding double-rendered output. Unmount and route switching abort only the reader. Stop uses the cancellation endpoint, including when pressed during creation.
+
+## Validation
+
+```sh
+.venv/bin/python -m pytest tests/unit/test_chat_runs.py tests/unit/test_agent_runtime.py -q
+cd raggr-frontend
+node --test tests/runs*.test.cjs
+npx tsc --noEmit
+```
+
+Tests cover real SQLite creation/claim contention, owner/conversation isolation and JWT endpoints, replay order/cursor, once-only answer persistence, request idempotency, application context, disconnect versus cancellation, file database close/reopen and lease expiry, drain, the actual shared runtime and legacy endpoint, SSE framing/duplicate suppression, authenticated discovery/cancel and cursor retry.
+
+The stall/metrics tests run through the durable worker: persistence happens in `runs.py`, and request cancellation detaches the subscriber rather than stopping generation.

docs/deployment.md

@@ -1,147 +1,68 @@
-# Deployment & Migrations Guide
+# Deployment
 
-This document covers database migrations and deployment workflows for SimbaRAG.
+Production Compose defines `postgres` (pgvector on PostgreSQL 16), optional `simbakit`, and two application services: `raggr-blue` on port 8080 and `raggr-green` on port 8081. Enable one application profile for a normal startup. The image builds the React frontend with `npm ci`, exports the frozen `uv.lock` dependency graph, and installs hashed Python packages system-wide so `python` and `aerich` work in startup and Compose exec commands. Runtime Node/MCP tools and uv are version pinned.
 
-## Migration Workflow
+## Configuration before deployment
 
-Migrations are managed by [Aerich](https://github.com/tortoise/aerich), the migration tool for Tortoise ORM.
+Copy `.env.example` to `.env`, then replace the database placeholders and configure:
 
-### Key Principles
+- `DATABASE_URL`: `postgres://` with host, database, user and a non-placeholder password. With the included database, its credentials must match `POSTGRES_USER`, `POSTGRES_PASSWORD`, and `POSTGRES_DB`.
+- `JWT_SECRET_KEY`: random signing secret of at least 32 characters.
+- `FERNET_KEY`: URL-safe base64 encoding of 32 random bytes. Required by encrypted model fields even without email. Preserve this key across deployments and restore it with encrypted database backups.
+- `OPENAI_API_KEY`: required because the OpenAI fallback client always initializes, including when `LLAMA_SERVER_URL` is set.
 
-1. **Generate migrations in Docker** - Aerich needs database access to detect schema changes
-2. **Migrations auto-apply on startup** - Both `startup.sh` and `startup-dev.sh` run `aerich upgrade`
-3. **Commit migrations to git** - Migration files must be in the repo for production deploys
-
-### Generating a New Migration
-
-#### Development (Recommended)
-
-With `docker-compose.dev.yml`, your local `services/raggr` directory is synced to the container. Migrations generated inside the container appear on your host automatically.
+Generate keys locally with `openssl rand -hex 32` for JWT and `uv run --frozen python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` for Fernet. Store their output securely; never include configuration values in CI logs.
 
 ```bash
-# 1. Start the dev environment
-docker compose -f docker-compose.dev.yml up -d
-
-# 2. Generate migration (runs inside container, syncs to host)
-docker compose -f docker-compose.dev.yml exec raggr aerich migrate --name describe_your_change
-
-# 3. Verify migration was created
-ls services/raggr/migrations/models/
-
-# 4. Commit the migration
-git add services/raggr/migrations/
-git commit -m "Add migration: describe_your_change"
+make validate-config
 ```
 
-#### Production Container
+Validation runs again in `startup.sh` **before migrations**. Failure exits nonzero and names the invalid settings without printing values. It rejects unsafe defaults, malformed database/service URLs, invalid Fernet keys, empty model names, nonpositive/nonfinite timeouts, invalid numeric/boolean settings, context budgets exceeding the window, and invalid checkpoint paths. Successful validation checks syntax/policy, not network connectivity or credential validity.
 
-For production, migration files are baked into the image. You must generate migrations in dev first.
+OIDC is optional, but if enabled configure issuer, client ID, client secret and public callback URI together. Production requires HTTPS issuer/callback URLs; register `https://your-host/api/user/oidc/callback` with the provider. With discovery disabled, also set the four manual endpoint variables. Unused optional integration secrets are not required by validation. The current agent module nevertheless constructs Tavily at import time and its SDK rejects an empty key; until that initialization is made optional, application startup also needs a valid Tavily key for web search. Configured Twilio, Mailgun and SendBlue webhook integrations require their signature secret. Signature validation must remain enabled in production even for unused channels, because their routes are still registered. Metrics remain disabled (403) with an empty `METRICS_TOKEN`; when enabled it must be at least 32 characters. `CRON_SECRET` and `EMAIL_HMAC_SECRET` follow the same length policy if supplied.
+
+`TAVILY_API_KEY` is the correct spelling. Compose and startup accept the legacy environment spelling `TAVILIY_API_KEY` when the canonical value is empty; migrate existing configuration to the canonical name. For local Python started directly, use the canonical name in `.env`.
+
+Compose forwards context window, prompt budget, response reserve, llama read timeout, stream stall timeout, metrics token, manual OIDC endpoints, rate limits, and checkpoint/summary paths. Defaults are 32768 context tokens, 16000 prompt tokens, 4096 reserved response tokens, 120 seconds llama read timeout, and 30 seconds stream stall timeout. Align context settings with the smallest backend context window, including fallback.
+
+`raggr-checkpoints` persists `/app/checkpoints` for both colors. The default checkpoint file is `checkpoints.db`; the default summary file is the checkpoint path plus `.summaries.db`. If overriding either path, keep it inside a persistent mounted directory and ensure its parent exists. Use separate files. Back up PostgreSQL, this volume, and encryption keys together.
+
+The Compose bind mounts include `./obvault` and `./token.json` (Google OAuth, read-only). Provision a token file when enabling Google integrations; otherwise an empty regular file can occupy the mount path. Do not allow Docker to silently create a directory at `token.json`. If SimbaKit is enabled, supply PetKit credentials; app validation does not validate that separate service.
+
+## Startup and migrations
 
 ```bash
-# If you need to generate a migration from production (not recommended):
-docker compose exec raggr aerich migrate --name describe_your_change
-
-# Copy the file out of the container
-docker cp $(docker compose ps -q raggr):/app/migrations/models/ ./services/raggr/migrations/
-```
-
-### Applying Migrations
-
-Migrations apply automatically on container start via the startup scripts.
-
-**Manual application (if needed):**
-
-```bash
-# Dev
-docker compose -f docker-compose.dev.yml exec raggr aerich upgrade
-
-# Production
-docker compose exec raggr aerich upgrade
-```
-
-### Checking Migration Status
-
-```bash
-# View applied migrations
-docker compose exec raggr aerich history
-
-# View pending migrations
-docker compose exec raggr aerich heads
-```
-
-### Rolling Back
-
-```bash
-# Downgrade one migration
-docker compose exec raggr aerich downgrade
-
-# Downgrade to specific version
-docker compose exec raggr aerich downgrade -v 1
-```
-
-## Deployment Workflows
-
-### Development
-
-```bash
-# Start with watch mode (auto-restarts on file changes)
-docker compose -f docker-compose.dev.yml up
-
-# Or with docker compose watch (requires Docker Compose v2.22+)
-docker compose -f docker-compose.dev.yml watch
-```
-
-The dev environment:
-- Syncs `services/raggr/` to `/app` in the container
-- Rebuilds frontend on changes
-- Auto-applies migrations on startup
-
-### Production (zero-downtime blue-green)
-
-The app runs as one of two identical compose services: `raggr-blue`
-(host port 8080) or `raggr-green` (host port 8081), gated behind compose
-profiles so only one color runs at a time. Caddy health-checks both ports and
-routes traffic to whichever is healthy.
-
-```bash
-# Pull latest and swap colors with zero downtime
-make redeploy
-
-# Or run the swap directly
-./scripts/deploy.sh
-
-# View logs / exec into the active color (Makefile resolves the color)
+# Build and start a selected color only, after configuration is ready:
+docker compose --profile blue build raggr-blue
+docker compose --profile blue up -d --wait raggr-blue
 make logs
 make migrate-history
 ```
 
-`scripts/deploy.sh` does the following:
+Startup validates configuration, runs `aerich upgrade`, optionally starts Obsidian sync in the background (`OBSIDIAN_SYNC_MODE`, default `bidirectional` so notes and tasks Simba writes reach your devices; containers take turns via a lock file in the shared vault), and replaces the shell with the Python process. **A migration failure stops startup; the application is never launched.** Obsidian login/setup failures still log a warning and allow the application to continue.
 
-1. Detects the running color via `docker compose ps`.
-2. Builds the inactive color.
-3. Starts it with `docker compose up -d --wait`, which blocks until the
-   container healthcheck (`GET /api/health`) passes.
-4. If the new color never becomes healthy, it is stopped and the old color
-   keeps serving (the script exits non-zero).
-5. Otherwise the old color is stopped with a 30s grace period so in-flight
-   SSE streams can finish, then removed.
-
-To target a color manually, enable its profile:
+An empty database without Aerich metadata may need explicit initialization. Inspect its state first. Run the migration tool in a one-off container because the app cannot start until migrations pass:
 
 ```bash
-docker compose --profile blue up -d raggr-blue
-docker compose --profile blue --profile green ps
+docker compose up -d postgres
+docker compose --profile blue run --rm --no-deps raggr-blue aerich init-db
 ```
 
-#### Caddy configuration
+Do not run `init-db` against an existing schema or fake migrations without a reviewed reconciliation plan. The pgvector extension initialization only runs on a fresh PostgreSQL volume; for an existing database, have the database owner ensure the `vector` extension exists before indexing. `aerich init-db` initializes ORM tables, not pgvector document indexing.
 
-Caddy needs both upstreams with active health checks:
+To generate and review changes, use `make migrate-new` and inspect `migrations/models/`. Manual application/history use `make migrate` and `make migrate-history`. The Makefile chooses the running color. Production migrations must be backward-compatible while both colors overlap; column removals and renames require staged changes. Make database backups before schema changes.
+
+## Blue/green rollout
+
+`./scripts/deploy.sh` builds the inactive color, starts it with `--wait`, and stops the old color only when the new container is healthy. A failed new color is stopped while the old color remains serving. `make redeploy` first performs `git pull`, then calls that script. These commands perform a real deployment and should be run only by the deployment operator.
+
+An external reverse proxy must route to both ports and check `/api/health`. For example:
 
 ```caddyfile
-ask.torrtle.co {
+simba.example.com {
   reverse_proxy host.docker.internal:8080 host.docker.internal:8081 {
-    lb_policy first          # prefer blue when both are healthy
-    lb_try_duration 5s       # retry on the other upstream during the swap
+    lb_policy first
+    lb_try_duration 5s
     lb_try_interval 250ms
     health_uri /api/health
     health_interval 2s
@@ -151,106 +72,12 @@ ask.torrtle.co {
 }
 ```
 
-#### One-time cutover from the old `raggr` service
+The proxy configuration is outside Compose. During rollouts both colors briefly share checkpoints and Obsidian files; avoid concurrent reindex operations and consider sync ownership when continuous Obsidian sync is enabled.
 
-The first deploy after this change lands needs manual steps, because the old
-`raggr` container still holds port 8080 and the deploy script doesn't know
-about it:
+The current `app.py` entry point runs Quart in debug mode. Production process serving/debug hardening and broader readiness semantics require separate application changes; configuration validation does not resolve those concerns. Docker's existing health check remains `/api/health`.
 
-```bash
-# 1. Update the Caddy config to the two-upstream block below and reload Caddy
-# 2. On the server:
-git pull
-docker compose --profile green up -d --build --wait raggr-green   # port 8081, no conflict
-docker stop -t 30 $(docker ps -q --filter name=raggr-1)           # old raggr container
-docker rm $(docker ps -aq --filter name=raggr-1)
-# From here on, `make redeploy` handles swaps (green is now the active color)
-```
+## CI and release checks
 
-#### Blue-green caveats
+Gitea Actions runs backend Ruff, unit tests and deterministic context evaluation, plus frontend renderer tests, TypeScript checks and build. CI receives no production secrets and performs no deployment. Python resolves from `uv.lock` with frozen commands; npm uses `package-lock.json`. Lockfiles must match manifests; do not regenerate them during an image build or CI run.
 
-- **Migrations must be backward-compatible.** The new color runs
-  `aerich upgrade` on startup while the old color is still serving the
-  previous code. Additive changes (new tables/columns) are safe; destructive
-  changes (drop/rename a column the old code reads) need a two-step deploy:
-  ship code that stops using the column first, then drop it in a later deploy.
-- **Obsidian sync overlap.** During the ~30s swap window both containers run
-  pull-only `ob sync` against the shared `./obvault` mount. Both are
-  read-only pulls, so this is low risk, but avoid deploying mid-reindex.
-
-### Fresh Deploy (New Database)
-
-On first deploy with an empty database, `startup-dev.sh` runs `aerich init-db` instead of `aerich upgrade`. This creates all tables from the current models.
-
-For production (`startup.sh`), ensure the database exists and run:
-
-```bash
-# If aerich table doesn't exist yet
-docker compose exec raggr aerich init-db
-
-# Or if migrating from existing schema
-docker compose exec raggr aerich upgrade
-```
-
-## Troubleshooting
-
-### "No migrations found" on startup
-
-The `migrations/models/` directory is empty or not copied into the image.
-
-**Fix:** Ensure migrations are committed and the Dockerfile copies them:
-```dockerfile
-COPY migrations ./migrations
-```
-
-### Migration fails with "relation already exists"
-
-The database has tables but aerich doesn't know about them (fresh aerich setup on existing DB).
-
-**Fix:** Fake the initial migration:
-```bash
-# Mark initial migration as applied without running it
-docker compose exec raggr aerich upgrade --fake
-```
-
-### Model changes not detected
-
-Aerich compares models against the last migration's state. If state is out of sync:
-
-```bash
-# Regenerate migration state (dangerous - review carefully)
-docker compose exec raggr aerich migrate --name fix_state
-```
-
-### Database connection errors
-
-Ensure PostgreSQL is healthy before running migrations:
-
-```bash
-# Check postgres status
-docker compose ps postgres
-
-# Wait for postgres then run migrations
-docker compose exec raggr bash -c "sleep 5 && aerich upgrade"
-```
-
-## File Reference
-
-| File | Purpose |
-|------|---------|
-| `pyproject.toml` | Aerich config (`[tool.aerich]` section) |
-| `migrations/models/` | Migration files |
-| `startup.sh` | Production startup (runs `aerich upgrade`) |
-| `startup-dev.sh` | Dev startup (runs `aerich upgrade` or `init-db`) |
-| `app.py` | Contains `TORTOISE_CONFIG` |
-| `aerich_config.py` | Aerich initialization configuration |
-
-## Quick Reference
-
-| Task | Command |
-|------|---------|
-| Generate migration | `docker compose -f docker-compose.dev.yml exec raggr aerich migrate --name name` |
-| Apply migrations | `docker compose exec raggr aerich upgrade` |
-| View history | `docker compose exec raggr aerich history` |
-| Rollback | `docker compose exec raggr aerich downgrade` |
-| Fresh init | `docker compose exec raggr aerich init-db` |
+Before rollout, require these checks to pass, validate the deployment environment, verify persistent mounts, and ensure migrations support both versions. Frozen application dependencies do not pin OS apt packages or transitive dependencies of globally installed integration CLIs; stricter image reproducibility can additionally use image digests and separately locked tool bundles.

docs/development.md

@@ -1,258 +1,76 @@
-# Development Guide
+# Development
 
-This guide explains how to run SimbaRAG in development mode.
+The repository uses Python 3.13 with `uv.lock`, React 19 with Rsbuild, and PostgreSQL with pgvector. There is no `docker-compose.dev.yml` or `requirements.txt` workflow. Run the backend locally against the Compose PostgreSQL service.
 
-## Quick Start
+## Local setup
 
-### Option 1: Local Development (Recommended)
-
-Run PostgreSQL in Docker and the application locally for faster iteration:
+Install uv 0.7.2, Python 3.13, Node 22.14.0, and Docker Compose v2.
 
 ```bash
-# 1. Start PostgreSQL
-docker compose -f docker-compose.dev.yml up -d
-
-# 2. Set environment variables
-export DATABASE_URL="postgres://raggr:raggr_dev_password@localhost:5432/raggr"
-export CHROMADB_PATH="./chromadb"
-export $(grep -v '^#' .env | xargs)  # Load other vars from .env
-
-# 3. Install dependencies (first time)
-pip install -r requirements.txt
-cd raggr-frontend && yarn install && yarn build && cd ..
-
-# 4. Run migrations
-aerich upgrade
-
-# 5. Start the server
-python app.py
+cp .env.example .env
+uv sync --frozen --extra test
+cd raggr-frontend
+npm ci
+npm run build
+cd ..
+docker compose up -d postgres
 ```
 
-The application will be available at `http://localhost:8080`.
+Edit `.env` before starting the database. Set `POSTGRES_USER`, `POSTGRES_PASSWORD`, and `POSTGRES_DB`; use the same credentials in `DATABASE_URL`, but point its host at `localhost` for local Python. Use `postgres://` so both Tortoise and the pgvector URL conversion work. The PostgreSQL image creates the `vector` extension on an empty data volume using `docker/init-pgvector.sql`.
 
-### Option 2: Full Docker Development
+The backend needs `OPENAI_API_KEY` even with llama-server configured, a `JWT_SECRET_KEY`, and `FERNET_KEY` for encrypted model fields. Configure local writable paths, for example `CHECKPOINT_DB_PATH=/tmp/raggr/checkpoints.db` and `CONVERSATION_SUMMARY_DB_PATH=/tmp/raggr/summaries.db`, and create the parent directory. Keep optional integration credentials empty until needed.
 
-Run everything in Docker with hot reload (slower, but matches production):
+The backend loads `.env` with python-dotenv; exported process variables take precedence. Do not use `export $(cat .env | xargs)` to load secrets. Validate production policy with `make validate-config`, or check provided settings with relaxed development requirements:
 
 ```bash
-# Uncomment the raggr service in docker-compose.dev.yml first!
-
-# Start all services
-docker compose -f docker-compose.dev.yml up --build
-
-# View logs
-docker compose -f docker-compose.dev.yml logs -f raggr
+uv run --frozen python scripts/validate_config.py --development
+uv run --frozen aerich upgrade
+uv run --frozen python app.py
 ```
 
-## Project Structure
+The local entry point serves on `http://localhost:8080` in debug mode. A truly new database without Aerich metadata needs an explicit, reviewed `uv run --frozen aerich init-db` first. Do not use `init-db` or fake migrations as a routine repair for an existing database.
 
-```
-raggr/
-├── app.py                    # Quart application entry point
-├── main.py                   # RAG logic and LangChain agent
-├── llm.py                    # LLM client (Ollama + OpenAI fallback)
-├── aerich_config.py          # Database migration configuration
-│
-├── blueprints/               # API route blueprints
-│   ├── users/               # Authentication (OIDC, JWT, RBAC)
-│   ├── conversation/        # Chat conversations and messages
-│   └── rag/                 # Document indexing (admin only)
-│
-├── config/                   # Configuration modules
-│   └── oidc_config.py       # OIDC authentication settings
-│
-├── utils/                    # Reusable utilities
-│   ├── chunker.py           # Document chunking for embeddings
-│   ├── cleaner.py           # PDF cleaning and summarization
-│   ├── image_process.py     # Image description with LLM
-│   └── request.py           # Paperless-NGX API client
-│
-├── scripts/                  # Administrative scripts
-│   ├── add_user.py          # Create users manually
-│   ├── user_message_stats.py # User message statistics
-│   ├── manage_vectorstore.py # Vector store management
-│   ├── inspect_vector_store.py # Inspect ChromaDB contents
-│   └── query.py             # Query generation utilities
-│
-├── raggr-frontend/          # React frontend
-│   └── src/                # Frontend source code
-│
-├── migrations/              # Database migrations
-└── docs/                    # Documentation
-```
-
-## Making Changes
-
-### Backend Changes
-
-**Local development:**
-1. Edit Python files
-2. Save
-3. Restart `python app.py` (or use a tool like `watchdog` for auto-reload)
-
-**Docker development:**
-1. Edit Python files
-2. Files are synced via Docker watch mode
-3. Container automatically restarts
-
-### Frontend Changes
+## Frontend
 
 ```bash
 cd raggr-frontend
-
-# Development mode with hot reload
-yarn dev
-
-# Production build (for testing)
-yarn build
+npm ci
+npm run dev
 ```
 
-The backend serves built files from `raggr-frontend/dist/`.
-
-### Database Model Changes
-
-When you modify Tortoise ORM models:
+Rsbuild serves the development UI. The current Rsbuild configuration does not define an API proxy; use the combined build served by Quart for same-origin backend integration, or configure a development proxy before using API-backed flows. `npm run build` writes `dist`, which Quart serves in the combined application. TypeScript and renderer tests run independently of the bundler:
 
 ```bash
-# Generate migration
-aerich migrate --name "describe_your_change"
-
-# Apply migration
-aerich upgrade
-
-# View history
-aerich history
+node --test tests/*.test.cjs
+npx --no-install tsc --noEmit
+npm run build
 ```
 
-See [deployment.md](deployment.md) for detailed migration workflows.
-
-### Adding Dependencies
-
-**Backend:**
-```bash
-# Add to requirements.txt or use uv
-pip install package-name
-pip freeze > requirements.txt
-```
-
-**Frontend:**
-```bash
-cd raggr-frontend
-yarn add package-name
-```
-
-## Useful Commands
-
-### Database
+## Backend checks
 
 ```bash
-# Connect to PostgreSQL
-docker compose -f docker-compose.dev.yml exec postgres psql -U raggr -d raggr
-
-# Reset database
-docker compose -f docker-compose.dev.yml down -v
-docker compose -f docker-compose.dev.yml up -d
-aerich init-db
+make test
+make evaluate
+make lint
+make frontend-check
 ```
 
-### Vector Store
+Test commands supply synthetic OpenAI/Tavily keys to satisfy existing client constructors; no live API calls are needed. Unit tests use the existing test fixtures and generate an ephemeral Fernet key when needed. They do not invoke startup validation globally or require every integration secret. Configuration tests validate explicit synthetic environments, malformed settings, context budgets, signature policy, Compose forwarding, and startup failure ordering. Compose interpolation tests run when the Docker Compose CLI is installed; they neither start services nor contact a database.
+
+`make evaluate` runs local context-compaction fixtures without LLM calls. Gitea CI pins uv/Ruff/Python/Node and uses frozen Python dependencies and `npm ci`; TypeScript comes from the frontend lockfile. A lock/manifest mismatch must be fixed by the dependency owner rather than resolved during CI.
+
+## Changes and migrations
+
+Backend routes and models live in `blueprints/`; chat/tool routing and context management live in `blueprints/conversation/`. `config/db.py` holds the Tortoise config used by Aerich. Retrieval is in `blueprints/rag/`, and integration clients are in `utils/`.
 
 ```bash
-# Show statistics
-python scripts/manage_vectorstore.py stats
-
-# Index new documents from Paperless
-python scripts/manage_vectorstore.py index
-
-# Clear and reindex everything
-python scripts/manage_vectorstore.py reindex
+make migrate-new  # interactive script: container if running, local DB otherwise
+# Or, against your explicitly configured local database:
+uv run --frozen aerich migrate --name describe_your_change
+uv run --frozen aerich upgrade
+uv run --frozen aerich history
 ```
 
-See [vectorstore.md](vectorstore.md) for details.
+Review generated files in `migrations/models/`. Production blue/green rollouts require migrations compatible with both old and new code. See [deployment.md](deployment.md).
 
-### Scripts
-
-```bash
-# Add a new user
-python scripts/add_user.py
-
-# View message statistics
-python scripts/user_message_stats.py
-
-# Inspect vector store contents
-python scripts/inspect_vector_store.py
-```
-
-## Environment Variables
-
-Copy `.env.example` to `.env` and configure:
-
-| Variable | Description | Example |
-|----------|-------------|---------|
-| `DATABASE_URL` | PostgreSQL connection | `postgres://user:pass@localhost:5432/db` |
-| `CHROMADB_PATH` | ChromaDB storage path | `./chromadb` |
-| `OLLAMA_URL` | Ollama server URL | `http://localhost:11434` |
-| `OPENAI_API_KEY` | OpenAI API key (fallback LLM) | `sk-...` |
-| `PAPERLESS_TOKEN` | Paperless-NGX API token | `...` |
-| `BASE_URL` | Paperless-NGX URL | `https://paperless.example.com` |
-| `OIDC_ISSUER` | OIDC provider URL | `https://auth.example.com` |
-| `OIDC_CLIENT_ID` | OIDC client ID | `simbarag` |
-| `OIDC_CLIENT_SECRET` | OIDC client secret | `...` |
-| `JWT_SECRET_KEY` | JWT signing key | `random-secret` |
-| `TAVILY_KEY` | Tavily web search API key | `tvly-...` |
-
-## Troubleshooting
-
-### Port Already in Use
-
-```bash
-# Find and kill process on port 8080
-lsof -ti:8080 | xargs kill -9
-
-# Or change the port in app.py
-```
-
-### Database Connection Errors
-
-```bash
-# Check if PostgreSQL is running
-docker compose -f docker-compose.dev.yml ps postgres
-
-# View PostgreSQL logs
-docker compose -f docker-compose.dev.yml logs postgres
-```
-
-### Frontend Not Building
-
-```bash
-cd raggr-frontend
-rm -rf node_modules dist
-yarn install
-yarn build
-```
-
-### ChromaDB Errors
-
-```bash
-# Clear and recreate ChromaDB
-rm -rf chromadb/
-python scripts/manage_vectorstore.py reindex
-```
-
-### Import Errors After Reorganization
-
-Ensure you're in the project root directory when running scripts, or use:
-
-```bash
-# Add project root to Python path
-export PYTHONPATH="${PYTHONPATH}:$(pwd)"
-python scripts/your_script.py
-```
-
-## Hot Tips
-
-- Use `python -m pdb app.py` for debugging
-- Enable Quart debug mode in `app.py`: `app.run(debug=True)`
-- Check API logs: They appear in the terminal running `python app.py`
-- Frontend logs: Open browser DevTools console
-- Use `docker compose -f docker-compose.dev.yml down -v` for a clean slate
+For dependency changes, use `uv add package-name` for backend dependencies and `npm install package-name` in `raggr-frontend` for frontend dependencies. Review and include the corresponding lockfile changes. Use npm consistently for the frontend CI/build path; a `yarn.lock` also exists, but is not used by production or CI.

docs/immich-mcp/design.md

@@ -0,0 +1,432 @@
+# Immich MCP: technical design
+
+Status: proposed · Owner: Ryan · Date: 2026-10-13
+
+## 1. Summary
+
+Give Simba (and Claude Code) a small set of tools for the household Immich
+library so a request like *"my Maine trip, here are the photos"* turns into
+either a link to the album that already exists or a confirmed, one-step
+album creation. The tools live in a stdio MCP server we write and keep in
+this repository (`immich_mcp/`). raggr loads it the same way it loads
+Homebox and BookStack; Claude Code can register the same server with
+`claude mcp add`.
+
+Work ships in two phases:
+
+- **Phase 1, read-only.** Find albums, search photos by date, place and person,
+  CLIP search, and spot photos missing from an existing album.
+- **Phase 2, writes.** Create an album and add photos to an album, behind a
+  server-side switch (`IMMICH_ENABLE_WRITES=true`) and a count check that
+  stops the model from writing something the user did not confirm.
+
+There are no delete, archive, trash, metadata-edit or sharing tools in either phase.
+
+## 2. Goals and non-goals
+
+### Goals
+
+1. **"Do I already have an album for X?"** returns the album name, photo count,
+   date span and a clickable link, in one tool call.
+2. **"Photos from my Maine trip"** with no album returns a summary the user can
+   check (how many, which days, which towns, a few sample filenames) without
+   moving hundreds of asset IDs through the model's context.
+3. **"Make an album for it"** creates the album with exactly the photos the user
+   saw in that summary, or does nothing and explains why.
+4. **"Is the Maine album complete?"** lists photos taken during the album's dates
+   that are not in it, such as camera imports or a partner's phone, and can add them after confirmation.
+5. The same server works from Claude Code on the laptop.
+6. Tool results stay small enough for the local Qwen model's 32k context: each
+   result is at most about 2,500 characters, and all six tool schemas together
+   cost at most about 1,500 tokens.
+
+### Non-goals
+
+- Deleting, trashing, archiving, hiding, or editing assets or albums. Removing photos from an album is out too.
+- Showing images inline in chat. The desktop client's `photo` artifact
+  (`docs/desktop-client.md`) can build on this later using the asset links
+  these tools return.
+- Shared links, inviting album users, face naming, tagging.
+- Replacing `scripts/index_immich.py`, the one-off script that describes Simba photos into the
+  vector store. It is left untouched; retiring it is an open question (§13).
+- Non-admin access. Photos are household-private, so like Homebox and BookStack
+  these tools exist only on the admin agent.
+
+## 3. Example conversations (acceptance scenarios)
+
+These double as the manual verification script for the final ticket.
+
+**A. The album already exists**
+
+> **User:** my maine trip, here are the photos
+> **Simba** → `immich_find_albums(query="maine")`
+> → `[{"name": "Maine 2026", "asset_count": 412, "start": "2026-08-02", "end": "2026-08-09", "url": "https://photos.example/albums/…"}]`
+> **Simba:** You already have **Maine 2026**: 412 photos, Aug 2–9. [link]
+
+**B. No album yet**
+
+> **User:** find my maine trip photos
+> `immich_find_albums("maine")` → `[]`
+> `immich_search_photos(state="Maine")` → total 398; clusters: Aug 2–9 2026 (391), Jul 14 2023 (7); top places Bar Harbor, Portland
+> **Simba:** I found 391 photos from Aug 2–9, 2026, mostly Bar Harbor and Portland, plus 7 from a day in July 2023. Want an album for the August trip?
+> **User:** yes, call it Maine 2026
+> `immich_search_photos(taken_after="2026-08-02", taken_before="2026-08-09")` → total 455 (this adds the drive up and photos with no GPS)
+> **Simba:** Counting everything from Aug 2–9, not only photos tagged in Maine, gives 455 (64 have no location or were taken on the way). Create **Maine 2026** with all 455?
+> **User:** yes
+> `immich_create_album(name="Maine 2026", criteria={taken_after, taken_before}, expected_count=455)` → created, 455 added, link
+
+**C. The album exists but is incomplete**
+
+> **User:** is the maine album missing anything?
+> `immich_find_albums("maine")` → Maine 2026, Aug 2–9
+> `immich_album_gaps(album_id=…)` → 37 photos taken Aug 1–10 are not in the album (mostly Aug 5, Canon EOS R6)
+> **Simba:** 37 photos from those days aren't in it, mostly camera shots from Aug 5. Add them?
+> `immich_add_to_album(album_id, criteria={taken_after: 2026-08-01, taken_before: 2026-08-10}, expected_new_count=37)`
+
+**D. Content search**
+
+> **User:** pictures of simba in the snow
+> `immich_smart_search(query="orange cat in snow", person="Simba")` → top 10, with dates and links
+
+## 4. Approach and alternatives
+
+| Option | Pros | Cons |
+|---|---|---|
+| **A. Our own stdio MCP server in this repo (chosen)** | One implementation serves raggr and Claude Code; we control tool names, descriptions and output size; write safety is enforced in code we own; no new runtime dependency (`mcp` 1.30 is already locked through `langchain-mcp-adapters`); fits the Homebox/BookStack loading pattern | We maintain it (≈500 lines plus tests) |
+| B. Third-party Immich MCP server (several exist: lidless-labs `immich-mcp`, `barryw/ImmichMCP`, `okopok/immich-mcp`, …), pinned and allowlisted like Homebox/BookStack | No code to write | Not evaluated in depth. Generic API-wrapping servers return per-asset JSON (a 400-photo trip won't fit in 32k) and add assets by ID list (the model would have to copy hundreds of UUIDs). Our key, which needs album write permissions, would go to a young third-party package. Tool descriptions we can't edit cost context. |
+| C. Plain LangChain tools in `utils/immich_service.py` (like Mealie/YNAB) | Simplest wiring | Simba only; Claude Code would need a second implementation |
+
+Option A keeps option C's control and adds reuse from Claude Code. Most of
+the work goes into compact summaries and write guards, and no off-the-shelf
+server gives us either.
+
+## 5. Architecture
+
+```
+            ┌───────────── raggr (admin agent) ─────────────┐
+            │ app.py lifespan                                │
+            │   load_immich_tools() ─┐                       │
+            │                        │ langchain-mcp-adapters│
+            │   wrap_admin_only()    │ (new stdio session    │
+            │   prefix immich_*      │  per tool call)       │
+            └────────────────────────┼───────────────────────┘
+                                     │ stdio
+ Claude Code ── claude mcp add ──────┤
+                                     ▼
+            ┌──────── immich_mcp (python -m immich_mcp) ─────┐
+            │ server.py   FastMCP tools (6)                  │
+            │ criteria.py PhotoCriteria → Immich search body │
+            │ summarize.py clusters / places / samples       │
+            │ client.py   ImmichClient (httpx, method+path   │
+            │             allowlist, no DELETE)              │
+            └────────────────────────┬───────────────────────┘
+                                     │ HTTPS, x-api-key
+                                     ▼
+                              Immich server (/api)
+```
+
+**Process lifetime.** `MultiServerMCPClient.get_tools()` without a session
+creates tools that open a **new stdio session for every call**
+(`langchain_mcp_adapters/tools.py`). The server therefore must be
+**stateless between calls**: no in-memory selections, caches or tokens. Write
+tools take the same *criteria* as the preview and resolve them again, and
+the `expected_count` check (§8) catches any drift. Starting a process costs
+roughly 0.5–1 s per call. That is acceptable for this use, and IMM-04 measures it.
+
+## 6. Package layout
+
+New top-level package (copied into the image by a new `COPY immich_mcp ./immich_mcp`
+line, because the Dockerfile copies directories explicitly):
+
+```
+immich_mcp/
+  __init__.py        # docstring only
+  __main__.py        # `python -m immich_mcp` → server.main()
+  config.py          # env parsing (§10), ImmichConfig dataclass
+  client.py          # ImmichClient: async httpx, allowlisted endpoints only
+  criteria.py        # PhotoCriteria (pydantic), date/time-zone handling, request bodies
+  summarize.py       # pure functions: clusters, places, samples, histogram
+  server.py          # FastMCP("immich"), tool definitions, JSON output shaping
+tests/unit/
+  test_immich_client.py
+  test_immich_criteria.py
+  test_immich_summarize.py
+  test_immich_server.py      # tools via an in-memory MCP session
+  test_immich_mcp.py         # raggr loader + admin gate (mirrors test_homebox_mcp.py)
+```
+
+raggr-side changes:
+
+- `utils/mcp_tools.py`: `load_stdio_tools(..., args: list[str] | None = None,
+  cwd: str | None = None)`. Today `args` is hard-coded to `[]` and no `cwd` is passed.
+  `StdioConnection` already supports both. Homebox and BookStack keep the defaults.
+- `utils/immich_mcp.py`: `IMMICH_TOOL_ALLOWLIST`, `select_immich_tools`,
+  `load_immich_tools()` (mirrors `utils/homebox_mcp.py`). The command defaults to
+  `sys.executable` (the app's venv interpreter) with `args=["-m", "immich_mcp"]`
+  and `cwd` set to the repo root (`Path(__file__).resolve().parents[1]`), so
+  `-m` finds the package whatever the parent's working directory is. The subprocess
+  env is only the `IMMICH_*` variables plus `PATH` (`load_stdio_tools` behaviour),
+  so no other secrets leak into it. `IMMICH_MCP_COMMAND` overrides the
+  command for debugging. Note the two names: `utils/immich_mcp.py` is raggr's
+  *loader*, and top-level `immich_mcp/` is the *server*. Neither imports the other.
+- `app.py`: add `await load_immich_tools()` to `admin_tools`.
+- `blueprints/conversation/tool_routing.py`: add a `"photos"` group with the terms
+  `immich photo photos picture pictures pic pics album albums trip trips vacation`,
+  map the prefix `("immich_", "photos")`, and add `"immich"` to the admin prefix
+  tuple in `ToolRouter.__init__`.
+- `blueprints/conversation/tool_labels.py`: `("immich_", "Looking through photos…")`.
+- `raggr-frontend/src/components/ActivityTimeline.tsx`: `["immich", Images]` (lucide).
+- `blueprints/conversation/prompts.py`: one sentence in INTEGRATIONS (§9).
+- `blueprints/integrations/__init__.py`: admin row `("Immich", ("IMMICH_URL", "IMMICH_API_KEY"))`.
+- `config/validation.py`: `IMMICH_URL` and `IMMICH_PUBLIC_URL` in the URL list;
+  `IMMICH_ENABLE_WRITES` in the true/false list.
+- `.env.example`, `docker-compose.yml`: env vars (§10). `IMMICH_URL` and
+  `IMMICH_API_KEY` already exist in `.env.example` for the indexer, so reuse them.
+
+## 7. Tool specifications
+
+Every tool returns **compact JSON** (`json.dumps(..., separators=(",", ":"),
+ensure_ascii=False)`), at most `MAX_RESULT_CHARS = 2500`. Dates in output are
+calendar dates (`YYYY-MM-DD`): a photo's date is its own local day (§7.4), and
+album `start`/`end` (UTC timestamps in Immich) are converted to
+`IMMICH_TIMEZONE`. If a result would go over the cap, lists are cut down (samples
+first, then places, then clusters) and `"trimmed": true` is set; the result is
+never cut mid-JSON. Links are built
+from `IMMICH_PUBLIC_URL`: albums at `/albums/{id}`, photos at `/photos/{id}`.
+Errors come back as `{"error": "<plain sentence>"}` and never raise, so the
+model can tell the user what went wrong.
+
+### 7.1 Shared input: `PhotoCriteria`
+
+| Field | Type | Meaning |
+|---|---|---|
+| `taken_after` | `YYYY-MM-DD` | inclusive, local day start |
+| `taken_before` | `YYYY-MM-DD` | **inclusive**, local day end (converted to the next day 00:00 local, exclusive) |
+| `state` | str | reverse-geocoded state/province, e.g. "Maine" |
+| `city` | str | e.g. "Bar Harbor" |
+| `country` | str | e.g. "United States of America" |
+| `person` | str | Immich person name, resolved via `GET /search/person?name=`; several matches → error that lists them |
+| `media` | `"photo" \| "video" \| "all"` | default `"all"` |
+| `favorites_only` | bool | default false |
+
+**The fields are flat tool parameters, never a nested object.** Every tool that
+takes criteria declares these eight parameters directly. A nested `criteria`
+object would produce `$defs`/`$ref` in the tool schema, which the local model's
+llama-server grammar handles poorly, and it costs more schema tokens. In the
+examples in §3, `criteria={…}` is shorthand for passing these parameters.
+
+For `immich_search_photos` and both write tools, at least one field must be
+set, so a request can never match the whole library. (`immich_smart_search` is
+already scoped by its query.)
+`taken_before < taken_after` is an error. Place names are matched
+case-insensitively: before searching, the server looks the value up in
+`GET /search/suggestions?type=state|city|country` and uses Immich's exact
+spelling. If nothing matches, it returns the three closest suggestions
+(`difflib.get_close_matches`) instead of searching.
+
+### 7.2 Phase 1: read-only
+
+**`immich_find_albums(query: str = "", limit: int = 10)`**
+Lists albums (`GET /albums`, both owned and shared-with-me) and ranks them by
+case-insensitive substring match, then `difflib` ratio ≥ 0.6. An empty query
+returns the most recently updated. Output per album: `id`, `name`,
+`asset_count`, `start`, `end`, `shared`, `owned` (false for albums shared with
+the key's user; adding to those may fail), `url`. Why match locally: Immich's `name` filter
+is an exact match, and "maine" should find "Maine 2026" and "Acadia / Maine".
+
+**`immich_search_photos(criteria…, samples: int = 5)`**
+1. `POST /search/statistics` gets the exact `total`.
+2. Pages `POST /search/metadata` (size 1000, `withExif: true`) up to
+   `MAX_SCAN = 5000` assets.
+3. Returns `total`, `scanned`, `truncated`, `first`, `last`,
+   `clusters` (§7.4, at most 8), `top_places` (city/state with counts, at most 5),
+   `no_location` (count with no city/state), `by_media` (photo/video counts),
+   `samples` (evenly spaced across the scan: date, city, filename, url), and
+   `criteria` (the normalized criteria, echoed so the model can pass it
+   straight to a write tool).
+
+**`immich_smart_search(query: str, criteria… (all optional), limit: int = 10)`**
+`POST /search/smart` (CLIP), with the same filters. Returns the top `limit` (at most 25)
+results in rank order: date, city, filename, url. There is no total, because
+smart search ranks results rather than filtering them. Use it to find content
+("in the snow", "sunset at the beach"), not to collect a trip.
+
+**`immich_album_gaps(album_id: str, pad_days: int = 1)`**
+Loads the album (`GET /albums/{id}` for `startDate`/`endDate`). It then collects
+the IDs already in the album (`POST /search/metadata` with `albumIds=[id]`,
+paged) and the photos in `[start − pad, end + pad]` (local dates), and returns the
+search summary from §7.2 computed over the photos **not** in the album. It also
+returns `criteria` for that window and `missing_count`, ready for
+`immich_add_to_album`. Returns an error if the album has no dates (it's empty).
+`missing_count` must be computed exactly as `immich_add_to_album` computes
+`expected_new_count`; both go through one shared helper,
+`resolve_missing(album_id, criteria)`. If the window has more than `MAX_SCAN`
+photos, `missing_count` is marked `truncated`, and a later add with that
+count fails the check. That's the safe direction.
+
+### 7.3 Phase 2: writes (only registered when `IMMICH_ENABLE_WRITES=true`)
+
+**`immich_create_album(name: str, expected_count: int, <criteria fields>, description: str = "")`**
+1. If any album already has this exact name (case-insensitive), return an error that names it
+   and suggests `immich_add_to_album`. Never create a duplicate.
+2. Resolve the criteria to asset IDs (full paging, no `MAX_SCAN` limit, but
+   refuse if `total > MAX_WRITE = 2000`).
+3. If `len(ids) != expected_count`, return `{"error": …, "actual_count": n}`
+   and write nothing.
+4. `POST /albums` with `albumName`, `description`, and `assetIds` for the first
+   500. Add the rest with `PUT /albums/{id}/assets` in chunks of 500.
+5. Return `id`, `name`, `added`, `url`.
+
+**`immich_add_to_album(album_id: str, expected_new_count: int, <criteria fields>)`**
+1. Resolve the criteria to IDs (same `MAX_WRITE` limit) and subtract the IDs already in the album.
+2. If the count of new IDs differs from `expected_new_count`, return an error and write nothing.
+3. `PUT /albums/{id}/assets` in chunks of 500. Count per-ID results from
+   `BulkIdResponseDto`: `success`, `duplicate`, other errors (with reasons).
+4. Return `added`, `already_present`, `failed` (at most 5 reasons), `url`.
+
+### 7.4 Trip clustering (`summarize.cluster_by_day`)
+
+Group scanned assets by the photographer's local day: the date part of
+`localDateTime`, taken **as-is**. Immich stores it as a "timezone-agnostic
+timestamp" of the wall-clock time, so converting it would shift days. If
+`localDateTime` is missing, fall back to `fileCreatedAt` converted to
+`IMMICH_TIMEZONE`. Sort the
+days and start a new cluster wherever the gap between consecutive days is more than
+`gap_days = 2` (one quiet day in the middle of a trip does not split it). Each
+cluster reports `start`, `end`, `count`, and its top city/state. Clusters are sorted
+by count descending; keep 8. This is how "my Maine trip" becomes
+"Aug 2–9, 2026" rather than "every photo ever taken in Maine". It's a pure function,
+tested with hand-built fixtures.
+
+## 8. Write safety
+
+These layers stack; any one of them blocks an unwanted write.
+
+1. **Not registered by default.** Write tools are defined only when
+   `IMMICH_ENABLE_WRITES=true`. Phase 1 can be deployed and used before
+   anyone decides to allow writes.
+2. **Admin-only.** The tools are compiled only into `app.admin_agent`, and each call
+   re-checks `is_admin` through `wrap_admin_only` (`utils/mcp_tools.py`).
+3. **Confirmed count.** `expected_count` / `expected_new_count` must equal the
+   number of photos the server resolves at write time. The model can only know the
+   number by having shown a search result, and the prompt (§9) tells it to get a yes
+   from the user first. If the library changed in between, the check fails.
+4. **Bounded size.** `MAX_WRITE = 2000` per call.
+5. **No destructive endpoints in the client.** `ImmichClient` exposes named
+   methods only, and its request helper accepts only the methods and path patterns
+   in `ALLOWED_ENDPOINTS`. A unit test asserts that no `DELETE`, `PATCH`, or
+   `/trash`, `/assets` (bulk) write is reachable.
+6. **A scoped API key.** Immich API keys carry permissions. The key needs only
+   `album.read`, `asset.read`, `asset.statistics`, `person.read`, and for Phase 2
+   `album.create` and `albumAsset.create`. Do not give it `all`.
+7. **No duplicate names.** See `immich_create_album` step 1.
+8. **Writes add and never remove.** At worst a write adds wrong photos to an
+   album. That is fixable in the Immich UI and loses no data.
+
+## 9. Prompt and routing
+
+Append to INTEGRATIONS in `blueprints/conversation/prompts.py`:
+
+> If immich_* tools are available, use them for photos and albums: call
+> immich_find_albums before searching or creating; for a trip, search by place,
+> read the date clusters, then search the whole date range so photos without
+> GPS are included; show the count and dates and get an explicit yes before
+> immich_create_album or immich_add_to_album, passing the confirmed count.
+
+Routing: the `"photos"` group (§6) makes these tools discoverable in
+`TOOL_ROUTING_MODE=dynamic`. In static mode they are always bound for admins, like
+Homebox.
+
+## 10. Configuration
+
+| Variable | Required | Default | Notes |
+|---|---|---|---|
+| `IMMICH_URL` | yes | – | Base URL the **server** uses (LAN/compose network). Already in `.env.example`. |
+| `IMMICH_API_KEY` | yes | – | Scoped key (§8.6). Already in `.env.example`. |
+| `IMMICH_PUBLIC_URL` | no | `IMMICH_URL` | Base for links shown to the user. |
+| `IMMICH_TIMEZONE` | no | `TZ`, then `America/New_York` | Local-day interpretation (§7.1, §7.4). |
+| `IMMICH_ENABLE_WRITES` | no | `false` | Registers the Phase 2 tools. |
+| `IMMICH_MCP_COMMAND` | no | `sys.executable -m immich_mcp` | Override for debugging. |
+
+All of them go in `.env`, `.env.example` and the `raggr` service environment in
+`docker-compose.yml` (project rule). If `IMMICH_URL` or `IMMICH_API_KEY` is unset,
+`load_immich_tools()` logs at info level and returns `[]`, like Homebox.
+
+Claude Code registration (documented in IMM-07):
+
+```bash
+claude mcp add immich \
+  -e IMMICH_URL=http://192.168.1.5:2283 -e IMMICH_API_KEY=… \
+  -e IMMICH_PUBLIC_URL=https://photos.… -e IMMICH_ENABLE_WRITES=true \
+  -- uv run --directory ~/Programs/raggr python -m immich_mcp
+```
+
+## 11. Immich API compatibility
+
+Verified against the upstream OpenAPI spec at `3.3.0-rc.0`:
+
+- Endpoints used: `GET /server/version`, `GET /albums`, `GET /albums/{id}`,
+  `POST /albums`, `PUT /albums/{id}/assets`, `POST /search/metadata`,
+  `POST /search/smart`, `POST /search/statistics`, `GET /search/suggestions`,
+  `GET /search/person`.
+- **Deprecation.** In v3.2.0 Immich deprecated the flat search fields
+  (`takenAfter`, `takenBefore`, `state`, `city`, `country`, `personIds`, `albumIds`,
+  `type`, `page`, …) in favour of a nested `filter` object (`SearchFilter`, with
+  `DateFilter {gte, lt}`, `StringFilterNullable {eq, in}`, `IdsFilter {any, all, none}`)
+  and `cursor` paging. The flat fields still work in 3.3, and they are the
+  only option before 3.2. Mapping: `takenAfter/takenBefore` → `filter.takenAt
+  {gte, lt}`, `state/city/country` → `{eq}`, `personIds` → `{all}`, `type` →
+  `{eq}`, `isFavorite` → `{eq: true}`, `albumIds` → `{any}`, and in `album_gaps`, `{none}`.
+- **Decision.** Build request bodies in a single function,
+  `criteria.to_search_body(criteria, version)`. Use the flat fields below 3.2 and
+  `filter`/`cursor` from 3.2 on, choosing by `GET /server/version` (read once per
+  process; a process lives for one call, so this is one extra request per call, or
+  `IMMICH_API_STYLE=legacy|filter` to skip it). Golden-body tests pin both shapes.
+  The `filter` style also gives `album_gaps` a server-side `albumIds: {none: [id]}`,
+  so it doesn't need to diff locally.
+- Production's Immich version is not yet known (it was unreachable from the design
+  machine). IMM-01 records it. If it is below 2.0, stop and revisit this design.
+
+## 12. Testing
+
+- **Pure units:** criteria validation, local-day to UTC conversion
+  (DST edges, inclusive `taken_before`), request-body golden tests for both API
+  styles, clustering fixtures (single trip, quiet day inside, two trips, empty),
+  sample spacing, output caps (a large fixture stays at most 2500 chars).
+- **Client:** `httpx.MockTransport`. Checks the endpoint allowlist (DELETE raises),
+  paging stops at `MAX_SCAN`, auth header, and that HTTP errors map to readable error strings.
+- **Tools:** call the FastMCP server in-process with
+  `mcp.shared.memory.create_connected_server_and_client_session` against the mock
+  transport. Covers scenarios A–D from §3, the `expected_count` mismatch (no write
+  happens), the duplicate-name refusal, write tools absent when the flag is off, and the
+  schema token budget (sum of `len(json.dumps(schema)) / 4` is at most 1500).
+- **raggr wiring:** mirror `tests/unit/test_homebox_mcp.py` (allowlist, admin
+  gate, missing env returns `[]`, prefix), plus `test_tool_routing.py` cases for the
+  `photos` group and admin hiding.
+- **Live (manual, IMM-08):** scenarios A–D against production with a scoped key.
+
+## 13. Open questions
+
+1. **Production Immich version** and whether `/search/statistics` and the
+   `filter` style are available (IMM-01).
+2. **Partner photos.** Immich search covers partners with "show in timeline"
+   enabled. Can the key's user add a partner's asset to their own album? If not,
+   `add` returns per-ID errors, which we report. Verify in IMM-01.
+3. **Time zone of typed dates.** Clustering uses each photo's own local day
+   (§7.4), but `taken_after`/`taken_before` filter on UTC `takenAt`, interpreted
+   in `IMMICH_TIMEZONE`. For a trip three hours west, a photo at 11 pm local
+   on the last day falls outside the window. `album_gaps` pads by a day, and
+   users can widen a range. A precise fix (filtering on `localDateTime`) is not
+   in the API. Accepted for now.
+4. **Retire `scripts/index_immich.py`?** Once smart search is live, the vector-store
+   copy of Simba photo descriptions may no longer be needed. Decide after Phase 1 has run for a while.
+5. **Sharing.** Should "send the Maine album to Mom" create a shared link? Not now;
+   it would need `sharedLink.create` and a separate design.
+
+## 14. Rollout
+
+1. Merge Phase 1 with `IMMICH_ENABLE_WRITES` unset. Deploy and run scenarios A, B (read
+   part), C (read part) and D.
+2. Create the Phase 2 key permissions, set `IMMICH_ENABLE_WRITES=true`, redeploy,
+   and run B and C end to end on a throwaway album name. Delete the test album by hand in Immich.
+3. Register the server in Claude Code.

docs/job-delivery.md

@@ -0,0 +1,103 @@
+# Scheduled and cron job delivery
+
+Priority2 uses durable delivery claims and attempt history. Apply migration 20
+(`migrations/models/20_20261012231017_application_foundation.py`) before running the
+updated backend. It adds `job_deliveries` and `job_attempts`, unique delivery
+identity `(kind, job_id, run_id)`, claim tokens/timestamps, scheduled-message
+`series_id`, unique `previous_occurrence_id`, `deleted_at`, and nullable cron
+`created_by_id`. Migration 20 does not assign owners to existing cron jobs.
+Downgrading drops delivery history and ownership/series fields.
+
+## Scheduled messages
+
+The panel preserves creation, cancellation and deletion. It polls the message
+list every ten seconds and exposes list/action errors, including permission or
+claim conflicts. Delivery history is loaded on demand with authenticated
+`GET /api/scheduled-messages/<id>/deliveries`. Refresh history to see later changes;
+a retry refreshes both the list and any open history, even when it fails.
+History includes each delivery's run ID, status/error, and every attempt's status,
+error, start time and finish time. An unfinished attempt is explicitly labelled.
+
+| Status | Meaning / action |
+| --- | --- |
+| `pending` | Waiting for its scheduled time; cancel or delete is available. |
+| `processing` | A worker claimed the delivery; retry is unavailable. |
+| `sending` | Attempt/history status: provider call is underway. |
+| `sent` | Provider acknowledged delivery; retry is unavailable. |
+| `failed` | Failure known to be safe to retry; explicit retry is available only when `retryable: true`. |
+| `uncertain` | Delivery might already have happened. Reconcile with provider records or the recipient before considering any resend. |
+| `cancelled` | Future work cancelled; delete is available. |
+
+Retry uses authenticated `POST /api/scheduled-messages/<id>/retry` with no schedule
+or status mutation. It executes the existing delivery identity and records a new
+attempt. A retry can return HTTP 409 with a changed message status (including
+`uncertain`); the UI surfaces the error and reloads the state. Editing a failed
+message's scheduled time does not retry it. The frontend update service accepts
+only `cancelled` as a status mutation and rejects `pending` at runtime, so updating
+status cannot bypass the explicit retry endpoint.
+
+Uncertain outcomes include ambiguous provider failures and abandoned claims.
+They are never automatically retried. This panel has no reconciliation or force
+resend action. Resolve the evidence through an operational reconciliation process;
+do not create a replacement message or new run ID simply to bypass uncertainty.
+
+Recurring occurrences have separate rows and delivery identities, linked by
+`series_id` and `previous_occurrence_id`. Cancelling a recurring occurrence cancels
+the series' future work. The panel labels this action **Cancel series**, including
+on completed or uncertain occurrences where future work must be stopped. Already
+started provider calls may still complete; cancellation cannot recall a delivered
+message, and attempt history remains evidence of the outcome.
+Deleting a pending/cancelled recurring occurrence uses the backend's soft delete
+and durable cancellation tombstone: the selected occurrence is hidden, and future
+series work is cancelled so recurrence repair cannot revive it. Other cancelled
+occurrences can remain visible. Nonrecurring pending/cancelled rows use the existing
+delete API. The UI never simulates cancellation/deletion by altering local series
+rows; it reloads the backend result.
+
+## Cron callers and ownership
+
+`POST /api/cron/<id>/trigger` requires `X-Cron-Secret` and a nonblank run identifier,
+maximum 255 characters. Supply either a JSON `run_id` or an `Idempotency-Key` header;
+the header takes precedence. Choose a stable identifier for each logical execution
+(for example the cron occurrence's UTC timestamp), and reuse it across repeated
+trigger requests for that execution. A new logical occurrence needs a new ID.
+Repeating a trigger with the same ID returns the existing outcome without resending;
+it does not retry a failed or uncertain delivery.
+
+```http
+POST /api/cron/<id>/trigger
+X-Cron-Secret: <configured secret>
+Idempotency-Key: daily-report:2026-10-13T12:00:00Z
+Content-Type: application/json
+
+{}
+```
+
+Legacy cron jobs require an explicitly assigned existing user. An administrator
+must use authenticated `PUT /api/cron/<id>` with
+`{"created_by":"<existing-user-uuid>"}` after reviewing the correct owner. Execution
+fails safely until ownership is assigned; it never selects an arbitrary admin.
+New jobs get the authenticated creator as owner. Agent execution uses that owner's
+context and permissions. Assigning ownership does not itself retry an earlier
+failed execution.
+
+Authenticated `GET /api/cron/<id>/deliveries` returns the same history structure.
+For a safe failed execution only, an administrator can use authenticated
+`POST /api/cron/<id>/retry` with `{"run_id":"<original-run-id>"}`. Reuse the failed
+run's ID; do not invent an ID to evade delivery claims. Processing, sending, sent,
+cancelled and uncertain executions are ineligible. Uncertain cron deliveries need
+the same reconciliation as scheduled messages. Database claims prevent duplicate
+workers from winning the same delivery; they cannot prove exactly-once external
+delivery when a provider acknowledgement is lost.
+
+## Frontend validation
+
+No additional packages are required. Tests use Node's test runner, the installed
+TypeScript transpiler and React's static renderer, following the existing renderer
+harness in `raggr-frontend/tests/history-management.test.cjs`.
+
+```sh
+cd raggr-frontend
+node --test tests/job-delivery*.cjs
+./node_modules/.bin/tsc --noEmit --target ES2020 --lib DOM,ES2020
+```

metrics.py

@@ -9,6 +9,20 @@ from prometheus_client import (
 
 REGISTRY = CollectorRegistry(auto_describe=True)
 
+AGENT_REQUESTS_TOTAL = Counter(
+    "raggr_agent_requests_total",
+    "Agent generation requests by channel and outcome.",
+    labelnames=("channel", "result"),
+    registry=REGISTRY,
+)
+AGENT_RESPONSE_SECONDS = Histogram(
+    "raggr_agent_response_seconds",
+    "Agent generation time by channel, excluding delivery.",
+    labelnames=("channel",),
+    buckets=(0.5, 1, 2, 5, 10, 30, 60, 120, 300),
+    registry=REGISTRY,
+)
+
 CHAT_RESPONSE_SECONDS = Histogram(
     "raggr_chat_response_seconds",
     "End-to-end chat response time in seconds.",

migrations/models/19_20261012225640_add_fk_indexes.py

@@ -0,0 +1,142 @@
+from tortoise import BaseDBAsyncClient
+
+RUN_IN_TRANSACTION = True
+
+
+async def upgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        ALTER TABLE "users" ALTER COLUMN "ldap_groups" TYPE JSONB USING "ldap_groups"::JSONB;
+        ALTER TABLE "beli_restaurants" ALTER COLUMN "cuisine" TYPE JSONB USING "cuisine"::JSONB;
+        ALTER TABLE "beli_restaurants" ALTER COLUMN "tags" TYPE JSONB USING "tags"::JSONB;
+        ALTER TABLE "artifacts" ALTER COLUMN "data" TYPE JSONB USING "data"::JSONB;
+        ALTER TABLE "artifacts" ALTER COLUMN "source_ref" TYPE JSONB USING "source_ref"::JSONB;
+        ALTER TABLE "ynab_changes" ALTER COLUMN "changes" TYPE JSONB USING "changes"::JSONB;
+        CREATE INDEX IF NOT EXISTS "idx_conversatio_user_id_1f0c4c" ON "conversations" ("user_id", "updated_at");
+        CREATE INDEX IF NOT EXISTS "idx_conversatio_convers_d1744c" ON "conversation_messages" ("conversation_id", "created_at");
+        CREATE INDEX IF NOT EXISTS "idx_user_memori_user_id_8278f0" ON "user_memories" ("user_id");
+        CREATE INDEX IF NOT EXISTS "idx_artifacts_message_d8728a" ON "artifacts" ("message_id");
+        CREATE INDEX IF NOT EXISTS "idx_artifacts_convers_188f03" ON "artifacts" ("conversation_id");"""
+
+
+async def downgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        DROP INDEX IF EXISTS "idx_conversatio_convers_d1744c";
+        DROP INDEX IF EXISTS "idx_conversatio_user_id_1f0c4c";
+        DROP INDEX IF EXISTS "idx_user_memori_user_id_8278f0";
+        DROP INDEX IF EXISTS "idx_artifacts_convers_188f03";
+        DROP INDEX IF EXISTS "idx_artifacts_message_d8728a";
+        ALTER TABLE "users" ALTER COLUMN "ldap_groups" TYPE JSONB USING "ldap_groups"::JSONB;
+        ALTER TABLE "artifacts" ALTER COLUMN "data" TYPE JSONB USING "data"::JSONB;
+        ALTER TABLE "artifacts" ALTER COLUMN "source_ref" TYPE JSONB USING "source_ref"::JSONB;
+        ALTER TABLE "ynab_changes" ALTER COLUMN "changes" TYPE JSONB USING "changes"::JSONB;
+        ALTER TABLE "beli_restaurants" ALTER COLUMN "cuisine" TYPE JSONB USING "cuisine"::JSONB;
+        ALTER TABLE "beli_restaurants" ALTER COLUMN "tags" TYPE JSONB USING "tags"::JSONB;"""
+
+
+MODELS_STATE = (
+    "eJztXQtv4za2/iuEscBOgTQ3j0lmmntxASfjTtPmMZhkbne2KTS0RNvcyJRWlDI1uv3vl6"
+    "QkWw9KlmxJFj0EtttG4qHkjxR5znce/HMwdyxk08Mrh7wgj0IfO2RwAf4cEDhH7D+k9w/A"
+    "ALru6i6/4MOxLQTMREtxB46p70HTZzcn0KaIXbIQNT3sRg8jgW3zi47JGmIyXV0KCP53gA"
+    "zfmSJ/hjx247ff2WVMLPQHovzP3wYBRZ6BLf6kwLWgjywD+oPfeTv32ZhgZFupHxQ2FdcN"
+    "f+GKa58+Xb/7UbTk7zE2TMcO5mTV2l34M4csmwcBtg65DL83RQR5/LGJ38dfP8IjvhT+FH"
+    "bB9wK0/A3W6oKFJjCwOUqD/5kExOTgAPEk/n+v/3dQAzc2BhxzTHwO0p9/hb9q9ZvF1QF/"
+    "1NVPw4+vTs+/E7/Sof7UEzcFIoO/hCD0YSgqAF8BKf6dg/JqBj05lHH7DJjsRTeBMb6wwn"
+    "E1uWIgY4A2Q20wh38YNiJTf8b+PDk7K4Hx/4YfBZKslYDSYRM+/Bzuolsn4T0O6QpC00Px"
+    "hM0B+Y7d8fEcycFMS2YgtSLRw/g/egow+w3WPbEX0UdQgu/j9e3o4XF4+4H/kjml/7YFRM"
+    "PHEb9zIq4uMldfnWeGYtkJ+PX68SfA/wT/vL8bZef+st3jPwf8nWDgOwZxvhrQSnyv8dUY"
+    "mNTAJlaimgObltQDu9OBjV4+8cHOICHIrrPsJUQ2Wvmit6o1fIOvaLzFfpFZ+Y6qLHxHxe"
+    "veUXbZS2zZVffhhMgWm/EmWG73KWy+9XIFZvIs3Xk5GHnwfnQ8hKfkF7QQEF6z94DElG25"
+    "kU73Keqmr6Ctrq7ewoNflzpdclKwX8d+E/LDj3H4cDV8NxoIDMfQfP4KPcsoAHOOKIVTRP"
+    "OAXkaSP/7yEdlLzVeOZVI/vg177OfaXIRt6gOFno8nTGXfEpRh1I1iSPBZ45w4idmSmkf5"
+    "W/OTefYKJGwGWNGz+ZNK5skacysxnapZXUZySrdsfaWeG36KCdVUW2GtW2E++kOiYz6yq3"
+    "Io4/aqWGFluuPoH48ptTHWOV7dDv/xXUp1vLm/ex83T+goVzf3l9ok20fNPW+SURfBZ5ne"
+    "xFX3EQnmOa0pNboJ8R1/OoNPD6OPF4ArP0/k4fr2cngBKJ6P4RN5vL+/YWg4jjA36ir6x1"
+    "UU/eNiRf84p+jjOduGjGe0qGMwpYQ6M5ma5IrOjk8qIMlaFUIp7qWxjDZ1g5mTll2LeMtL"
+    "KolqOwxcXn+pqpRIRJvUUBS1Tc2Mo2BLGzXrd+gtimuNVcl02dxo7d5A6w8L0Kp9xjmRWz"
+    "R3vMVAYpYl7h6UWWOCmJjzhrgDK0wbWW0bWewhPiK17KyEiDa1tKklUW7219TS3q+9GNgc"
+    "Lbwzv02FUcxtLT3Ujbv12+wMtDb9Nm3qfvfX764efOhLGfnVzVLNz8GWaVDerpreN/iAiM"
+    "UwBbx/YDtTTC7AHLoUMGUPfBE9fQEu9OAc+A7APgUffrkaAWZKYPazvMMs0bN1h0/kiTz4"
+    "bE5aABPwIZrg4BVxfOB6jokoBUK1XXwHqANg+AzWO/QB691jHTJtziHoiYztAP0Xk0YEsB"
+    "XB8QAkFphggukMiVb8lRyu1TJJbNvgBdqYL+5U/KwtNF8aj1RVjmQp0Iyytl7zbZIZOT55"
+    "W4W5O3lbTN3xe+U6sPtsIiOeJXWgzQmqog+3BvIKVPSHi9nXtYGmlJbsRlPqbo/tk2IU41"
+    "Cq8mpbZi9U3nBgcyrcbvQRoecVsFDr+SfNO6nPO/FxrBtmnZRRc6dtxdHjQkq/Op5kZl5i"
+    "Ar1Fge6SkMqAOV5EOn5fiXMZeJechUsuvAyyy+u74cfPciIvbp+ct5efH0fDrB4zh7hWVO"
+    "xSQEl9+6iap7zMVZ7zlX9lBhRlS7pBgvm4noYtEW3Ew9stqqdVQD0txvRUEn4Qeb7rQyoR"
+    "1ZDG362BCP95srXUcWwESck3n5TNrqhR3Eob+1Nd7afGknp/f5NaUi+vs76PT7eXI7YeCJ"
+    "hZIxyyXdd3j1JwZ3NoMrXrGUn852sW1IysgjP2+LzKypq1BxIL63l2xoaMXDD+FzIltlkx"
+    "oFk5BcFsRZFi1tLMcD3nBVv1ltScYHfa6cB2TGhvod5n4ruqLKtnxcvqWW5ZtS3oGlPPCV"
+    "xJRMfPD/d3ckgzYlnyAJs++A+wMW0tAn/w2+/tGE38N5c7nLO+5YyWyjvQDuc9Jmm0w3nv"
+    "BnYZ3VWRfJNHHTaYydVn8748gysZfLY5Ful4t/7N/EpQhJoxNE0nINsGTI54X8OwK4Uhoe"
+    "YMWQGzwoxmsh8f4v7UT30cIxtzLXWC7W1RuWRdfQh7UhgQnQvaln9HLCYDiYNnFJOjxR4e"
+    "sahVDDER3YHoS2f/9iF3OIhgjChKNR9FUkFmGShCeTgJRRYQLwVEw6/Yn4HTo+8tuAAe4u"
+    "3ZyxyCG0yeKY87uZp5zhy+u3wiE8cDL8zGZv+iCHrmDJjQhWNsYx9vHwiiXVENuaJiPlQG"
+    "6PrUIymwSnD9rXAoG9BR2zJRO/fqnVVympyVOE3O8k6TCVtFuDnB5rFkfypGMyunJqStzE"
+    "1mjRUCWpJW7qgIZ9cZD5Y0KrGcoYhl1OQmFOEiKsV4jR1rYdStt5ASUiTvtuvPQmA08+eS"
+    "CII1wMZCGlh5hpXQstkT2avV1NwkooqArOs2ar55U0eCWvHYerMuHsmI762ZrpaW0uUcYk"
+    "DyGNbOWlOXOj/IpK+lJ0mfMthSGBcRi4kxWMMvppwmVXnGSIbThBM8DTzBCwPO8l3fDj/w"
+    "ywQJuosWsY41ekhwkEspD1mcbmTvGLKQiJjewmUTDcTRvAeJPgBFvs8eTw+eCGc34254Ql"
+    "hAD8FD4LoOT2qbsymGXRvFLShwkSeqMGmOsiccZTRp69NBOUHNVMaMBaauDRdG3SyErJy2"
+    "HRI10Vxj5tBa7G9KSBV2rSsw+fqcB/OaFBAHKZkMlrg93eSHH063gFKUif3+5Pj1m9dvT8"
+    "9fv2VNxKssr7wpATcfaSww2CS7KCeoJ2NmMhbmGY1iPaSY08r1oAq6XRNbmBnaTB95kczd"
+    "0hSElFyH6QfLjb/H2Qc2pL6BPM+RRHUXT9m0lCLbfOc8rOYL95Mv1IHHezGwSlW6UoI37F"
+    "2pq12RhY2cUbKK9tsydFktLNMxUwsSVtYKmgDigfX2sOxMIUhap48TyBQxyGnw1pHImYFb"
+    "zyM/stvPFHA5XnGMLzpURJ6KIl2CdkVJojjPI9fugfPItxxm9g8FXKcNhfney2Tm7kF4MS"
+    "qAhqxlaCxbCSPmeMI6DDwE+K975hXYfAfQkD0GmNl+aI5Y/3bYMRcQWjMQVdlZc80i94RF"
+    "FhaNmLSbhGvlpRvQ7fplIfVIlavkChZDEkciB7LvpJArlIl2RxkebaGaN0wYxhgUeIMLAc"
+    "zJfZPo8e6RGXDiyYi2CYkSUwhikfg3iaX4JCMUNl+hsx3oRXrHi7RmUvaTSdFxWHsWh9V2"
+    "UIP6UVi5bGiJHS3LmC42pOX52rq4p9J2podM7GLpsTLF/veUkCre4S4i8ssOZl9/uuO2h7"
+    "Q3B+qA77gPw/ejCxDXIHwio9vh9c0FWBbN3P0R7vpEpDb8xDvIC97LGLvVflnfnMjKqmlQ"
+    "KGJAVDINi5w+FQ/uXUp3WG8w0upCJ8nhh9Hdu+u793kNZBDduQBueGrLE3kY3T1eAMoWyi"
+    "fyI1v1R+8uhGcBWU/kanh3NboRl0z+W+2ojurutwOmlgSehyL8NxmmdA8dDtXH5YMP76IJ"
+    "nBkkfpkBK460eceG5PMFsNiILJ7Ir6PRL/zPrwg9879v7+8ef+IX5myvm9mLfgxOQJHBJq"
+    "Nsty4NlkvJ6Vq9mVq93HkXm2N11KCcoCLbto6Z04SgjpnTA1vI9Maf3XhR9yD1rKDmexOY"
+    "NED5qh9Fl5sifeJ7rzyH/OyMBxKaN751UMbumqyR8S9nrEld9UnduvlUOo0qn3KaeLMaen"
+    "VGTGvVUq3a9Zy5W4u2XUmoMke7xlQ7crQjR1lHzoZHOe3kECdlsii9gGxg3mZEdUhaH+KG"
+    "y5wPZRHg23kddu2Va+GEI51drJlSTahppvTbG9i6xxq1yVUlzySR8FWZI0uKOavcMSnN8l"
+    "a/LROIxYNCffl3zWZtsudswWYlwK+hAKWltG2bOVtomwJBuR5UQbdrnUgAVVjFYc3cLS7l"
+    "0FeVs6vJ66EJWz1mRccTV53BuW4UgbnrWRwyAiFYG+mB0g40sdAHYkHbwm18MZrObJbO1N"
+    "zCXpigmlvY04HVlct05bI+Vy7zePmkwINbn7zMmamPy87UQrbVol0ZYApIvTR0a3i9zKA1"
+    "Te25K4JRPG4cUEwQpdFkS3jcPIyIZWRtcvG3yUZ+yg8m16zgTljBzKDV4lcysqpwWB2QLA"
+    "Uzviq6BeIdZlNtMVm7g7dukGZGTBHTuytOcLkS110GkoIqTtHjKkESx8VBEse5IIlvK3q4"
+    "lROhTezXmolxe/1RLychM3RmY8ebOU6tzScrpxFdzskAc4UnD+bPD/d3BdNyJZJlWLDpg/"
+    "8AG9PWrKCEHjoOsO1jQg/581pSRTkK5URwlvPN6Ky8g3ygOzaRwWwYWXZu8STOiCk5h5vf"
+    "l9SrwCAPyb4cje4uwBgh8kR+Hd49Go/3xuNHnsDPbF1mpRq+15OcfTYDnw3xA/OoF9YJTQ"
+    "ttVB10B3O34fqg1HQ8WU1QZOI5tOXALWWya20odBgJ9xK/EnDeja6ub4c3r84OTjKOnnhO"
+    "vs6roI4vq01b7JZcCiiyVnbtkfThVIJn8c4ft9fb/ibb/gT55mahEmlJ7dbql1tL+6H3Yl"
+    "zzfujIGVDTZZmW0l7LpE9lS8dlJgS8t/it9V+m50ifCkYMPR9PQodazlm3vHdQ5qaDUauK"
+    "J+oMAWsRmH7gIQv4jmMDNrHYWIBXIg2ZSYwDa4r8A8C1uafg5Ojk/DvWho0hl8AE+DME4o"
+    "cCFxJk50/dae0purBFN+vLQYnTz48C4aoSKXF7NanpVmjAZ/b2dSCM26sJ4fnrCgievy4E"
+    "kN/KWFLYl+1wJXMwFlATwXaqq7APvI45Grfv3hxtTctqxfCkTuBx5hhN6qCbltoK414xKa"
+    "1A/II8Ki0KVEiIJiS6OyvpeIvvv+lzp7TVvJ9Ws47e3ouBzdNcDuFrloiFrVtFMy/aLi/S"
+    "zrk/G5kt+XMK62GXltoCtvWbcC9R00kDOmmgo6SBotWuAfSuMt31dYlbX+02v5SvB7OwFP"
+    "1WOCaOJ+vp0rcWzfTingLyYfQI7j7d3OyKCP5M4Dg+ik/CBSdvH5TRwQvW0IgOxOtZFeFi"
+    "dqbJ2PS2DyVMEzOn5xV4mdOsFrmiZfgtXUa4ZaJLYFADwri9mhA2X7CPYRC6a/IglubSp+"
+    "Q6zKZvTStsMpvedmjt4gQrIY1mLlQtj2V5pNrGX3iv6NV2SiZBG0oP8CoNnUxIbR082a+Z"
+    "GkdPHp8dnFYOn2T4Qo8fVLERmBJpDSp/9e1glcprYJflodhTTLzZaZnyHnSFqR1XmApN3L"
+    "qaRkJKH/cn4ft2RxJcxamyBSzB8v7BWpogyrrFfTtDXhMFmxAFcQ61MfWcwK1ZLkAq3Jb9"
+    "qwTQmoFpkIGZYcuSVQkt3YNWQtraTaHJnoY8AiVBmKV4JsX0pq4JhBYJhKkDbaMu7ZoSUg"
+    "TYtpN/Q0ygJyVfSy3djOR+prPWMnETiBjiVHa5hbsWzpVwmXGrHKTMWM0dv6hNx300Hd9z"
+    "HX+wxn4MGx1UNSIjy0FbksoYONrlrA2eNjcUbfD0GVK9t+/Z3j4ThaqKNvVZXMZq3W4uGl"
+    "bNJXZdRKzvHWIvgO1MgTMBX2fQBwiaM0AXxARhd9ZBWEUEjBfAwpMJGywAp2x8qC/yfOeY"
+    "H2UhySJutv8n8kQ+3w0v/07B8MM1mEEKiANmmPpMfTkA1GGNMQXsf1xIPJW7szyLPxhZ2K"
+    "fgFTqcHgLIOoKU4imZI+KDqcMfOPGcOfjb66Mj4Dvgb7wc4SG49gFlNgMThOGbTLDH3om/"
+    "ehvZzJd4Wpj/JNV3JKlPkeqyU3UnTH364eTk9PTNydHp+duz12/enL09WuZA5W+VJUNdXr"
+    "/nn2vqs44XxGIliA/SRn7QlGA3WTVND8C+JNXkvaEUeS/IM57ZfRtZshjpso9IJt1dNqE6"
+    "X1XyEB4f+4vaVGBGTE1HVBUy8LSYDDyVHNAuUKln6KaElASy+fIB0JTnmRSjuJJQ08htvh"
+    "Ao+2moVhmQpYCaCLZTonqlbqdhLCkHvBLRhRjyVQJ6ZJLdCjdBgUV2G/sQ1hhkwtegWdXW"
+    "N5mmFshig6K2y6kpN1P3+3c9N5P2vTfqe8fEdGT8fam3eCWkg6HZHiCylmTsaHkCREJMwx"
+    "gqzS/SQzhKYUyKaRh5/TxjjIwN52ReWEPKptgUGc6EK1dIMjsL+Z+s2DdaV1/7jgZ75TsS"
+    "tsi6vIJ0o4NqhkvdFIPBB+R9LyRBHFcC4FzUMjgEX77Ey9iXL9xLI3xB3KHD2to2BU+DoX"
+    "DKIOtpkHcmNdqzNqjWGlSla8pyZX5TaFDxW9qg2sigWoZkbZgDojBX3Hz2h7YFtC3QHxh1"
+    "Zn4DIGr9db/01w9wgVCR3hreXK+vurydJtqV2aF1+HKLfkn2NEInyIsrydXUJAvEFSHs29"
+    "Yo9fazX9vPQzAWEz6KESnYhzKt1m9INCVQkT+5wQRRHq8KAXUZYiDRxX+LEFQXeoj4f6cr"
+    "GgRzguOBt5bxJo30qPfFzqKcSs7tWQ3cJut5SlJJcqCFQLJ5XJG1jk27FNLGGK/JO3fqeP"
+    "/j9oooE117/4UZU/PzTsooAmvbOtoOKdTOq2e3jaU2JrQxoY2JqsbEgpgPPvQL+axVgwom"
+    "BGtr0GXjnfBaoSOk5jefEtLavEybjyCqS3ZlxJRcRVvJ2O9JZtjOooM6TgwT1WI3zrHMS+"
+    "s6szuuMyuGZMJ+ZrjrbDim2Q70sPZhWBHPn69jpKelFNlkujDVe6RpPq7nrB9rEda12Wrt"
+    "R+21hhlHw1UNr4vb73l0neZ8Nee7+41EctJILaJyJaJm2EPzpVbZ7uY5L7WptKSYLtuWQn"
+    "RiwylHSqY7Fk/NtJQin3zrk3Mj6rwBxnzXPtvmGfOeesgUQ1E7yLSDTAlw8dx1PD8M763N"
+    "lkuFlQS2lTDReMLVhTUrpySirXghNpumen4WlFeKd5u6gOYElcS0lRmqAw4G21tJPaGBh8"
+    "jDprREVXTnoIz0has2vWF696j+7ZblGop53Rfk0ZrlDxMiitJFbSyF/NOoo/WEzdUE8LjS"
+    "/nxcsj8fS/Znh/hIxqWXlD9ciejyhz0rf/jX/wO/ZAiL"
+)

migrations/models/20_20261012231017_application_foundation.py

@@ -0,0 +1,218 @@
+from tortoise import BaseDBAsyncClient
+
+RUN_IN_TRANSACTION = True
+
+
+async def upgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        CREATE TABLE IF NOT EXISTS "chat_runs" (
+    "id" UUID NOT NULL PRIMARY KEY,
+    "request_key" VARCHAR(128) NOT NULL,
+    "payload" JSONB NOT NULL,
+    "active_key" VARCHAR(36) UNIQUE,
+    "status" VARCHAR(20) NOT NULL DEFAULT 'running',
+    "worker_token" UUID NOT NULL,
+    "lease_until" TIMESTAMPTZ NOT NULL,
+    "cancel_requested" BOOL NOT NULL DEFAULT False,
+    "sequence" INT NOT NULL DEFAULT 0,
+    "partial" TEXT NOT NULL,
+    "user_saved" BOOL NOT NULL DEFAULT False,
+    "answer_saved" BOOL NOT NULL DEFAULT False,
+    "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "conversation_id" UUID NOT NULL REFERENCES "conversations" ("id") ON DELETE CASCADE,
+    "owner_id" UUID NOT NULL REFERENCES "users" ("id") ON DELETE CASCADE,
+    CONSTRAINT "uid_chat_runs_owner_i_98309f" UNIQUE ("owner_id", "conversation_id", "request_key")
+);
+CREATE INDEX IF NOT EXISTS "idx_chat_runs_lease_u_b8a59a" ON "chat_runs" ("lease_until");
+        CREATE TABLE IF NOT EXISTS "chat_run_events" (
+    "id" BIGSERIAL NOT NULL PRIMARY KEY,
+    "sequence" INT NOT NULL,
+    "payload" JSONB NOT NULL,
+    "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "run_id" UUID NOT NULL REFERENCES "chat_runs" ("id") ON DELETE CASCADE,
+    CONSTRAINT "uid_chat_run_ev_run_id_1326b1" UNIQUE ("run_id", "sequence")
+);
+        CREATE TABLE IF NOT EXISTS "job_deliveries" (
+    "id" UUID NOT NULL PRIMARY KEY,
+    "kind" VARCHAR(20) NOT NULL,
+    "job_id" VARCHAR(36) NOT NULL,
+    "run_id" VARCHAR(255) NOT NULL,
+    "status" VARCHAR(20) NOT NULL DEFAULT 'pending',
+    "claim_token" UUID,
+    "claimed_at" TIMESTAMPTZ,
+    "error" TEXT,
+    "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CONSTRAINT "uid_job_deliver_kind_1add26" UNIQUE ("kind", "job_id", "run_id")
+);
+COMMENT ON TABLE "job_deliveries" IS 'Durable run identity; never reclaim ambiguous external deliveries.';
+        CREATE TABLE IF NOT EXISTS "job_attempts" (
+    "id" UUID NOT NULL PRIMARY KEY,
+    "claim_token" UUID NOT NULL UNIQUE,
+    "status" VARCHAR(20) NOT NULL DEFAULT 'processing',
+    "error" TEXT,
+    "started_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "finished_at" TIMESTAMPTZ,
+    "delivery_id" UUID NOT NULL REFERENCES "job_deliveries" ("id") ON DELETE CASCADE
+);
+        ALTER TABLE "conversations" ADD "pinned" BOOL NOT NULL DEFAULT False;
+        ALTER TABLE "scheduled_messages" ADD "deleted_at" TIMESTAMPTZ;
+        ALTER TABLE "scheduled_messages" ADD "previous_occurrence_id" UUID UNIQUE;
+        ALTER TABLE "scheduled_messages" ADD "series_id" UUID;
+        COMMENT ON COLUMN "scheduled_messages"."status" IS 'PENDING: pending
+PROCESSING: processing
+UNCERTAIN: uncertain
+SENT: sent
+FAILED: failed
+CANCELLED: cancelled';
+        ALTER TABLE "cron_jobs" ADD "created_by_id" UUID;
+        ALTER TABLE "scheduled_messages" ADD CONSTRAINT "fk_schedule_schedule_31982f9c" FOREIGN KEY ("previous_occurrence_id") REFERENCES "scheduled_messages" ("id") ON DELETE SET NULL;
+        ALTER TABLE "cron_jobs" ADD CONSTRAINT "fk_cron_job_users_b3e69ad6" FOREIGN KEY ("created_by_id") REFERENCES "users" ("id") ON DELETE SET NULL;"""
+
+
+async def downgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        ALTER TABLE "scheduled_messages" DROP CONSTRAINT IF EXISTS "fk_schedule_schedule_31982f9c";
+        ALTER TABLE "cron_jobs" DROP CONSTRAINT IF EXISTS "fk_cron_job_users_b3e69ad6";
+        ALTER TABLE "cron_jobs" DROP COLUMN "created_by_id";
+        ALTER TABLE "conversations" DROP COLUMN "pinned";
+        ALTER TABLE "scheduled_messages" DROP COLUMN "deleted_at";
+        ALTER TABLE "scheduled_messages" DROP COLUMN "previous_occurrence_id";
+        ALTER TABLE "scheduled_messages" DROP COLUMN "series_id";
+        COMMENT ON COLUMN "scheduled_messages"."status" IS 'PENDING: pending
+SENT: sent
+FAILED: failed
+CANCELLED: cancelled';
+        DROP TABLE IF EXISTS "chat_runs";
+        DROP TABLE IF EXISTS "job_attempts";
+        DROP TABLE IF EXISTS "chat_run_events";
+        DROP TABLE IF EXISTS "job_deliveries";"""
+
+
+MODELS_STATE = (
+    "eJztXQtv4za2/iuEscBOATc3iScz0+zFBZyMO007eSDx3G63KTSyTNtsZMqrR1Kj2/++JC"
+    "VZL1KWZEkWHQLzSCQeSvr4Ouc7h4d/9pbWFJrO0aWFn6Ht6C6ycO8c/NnD+hKSH7j3+6Cn"
+    "r1bRXXrB1ScmEzBiJdkdfeK4tm645OZMNx1ILk2hY9hoFTwMe6ZJL1oGKYjwPLrkYfRvD2"
+    "quNYfuAtrkxq+/kcsIT+Ef0KG//trzHGhraEqf5K2mugunmu72fqPlVk/aDEFzmvggvyi7"
+    "rrnrFbv25cvVx+9ZSfoeE82wTG+Jo9Krtbuw8Ka456HpEZWh9+YQQ5s+NvZ99PUDPMJL/q"
+    "eQC67twc03TKMLUzjTPZOi1PvfmYcNCg5gT6L/vP2/XgncSBtQzBF2KUh//uV/VfTN7GqP"
+    "Puryh+H9m8G7b9hXWo47t9lNhkjvLyaou7ovygCPgGT/Z6C8XOg2H8qwfApM8qJVYAwvRD"
+    "hGnSsEMgSoGmq9pf6HZkI8dxfk19OzsxwY/394z5AkpRiUFunw/nC4CW6d+vcopBGEhg3D"
+    "DpsB8iO546Il5IOZlExBOg1Ej8IfOgow+YbpLTbXwSDIwXd8dT16GA+v7+iXLB3n3yaDaD"
+    "ge0Tun7Oo6dfXNu1RTbCoBP1+NfwD0V/Cv25tRuu9vyo3/1aPvpHuupWHrRdOnsfEaXg2B"
+    "STRsbCYq2bBJSdWwe23Y4OVjA3ahYwzNMtNeTKTSzBe8Vanm673AyQ7rRWrmOy4y8R2L57"
+    "3j9LS3QgQRzjJ8YVkm1DEfx0goBeOESDU1DsoqLMUX3ovb28+JPn9xNU4h+OX6YnT/5oQB"
+    "Swohl12+uhmn4IxpQEXVmpjIDrpNla6528xSXZOh+uDsiavIUDCy4H1v2RDN8U9wzSC8Iu"
+    "+hY4OnwQQq8pegmq6CFl2N3sLWXzYqcrxTkK8j3wT9Hnc5fLgcfhz1GIYT3Xh60e2pJgBz"
+    "CR1Hn0OHM7oDye9/uofmxpDgYxk3N679Gru51ImwTa8armZ7eFdQSDX3HpYYCN120YyYgj"
+    "sCMQyqkQwJOnysUys2bBIDKntrebpMX9ExGQrT4Nn0STkDZosZHxtXxax5LT62G7bqE8/1"
+    "56SYyaOs+8atexf+wbFdxuQqH8qwvCzWfZ5NMvrnOKGahbrsm+vhP79JmCSfb28+hcVjmt"
+    "vl59sLZeofokWYNfWdFdSfeAokNQlH2Ftm1MdE68bE9zx0el8eRvfngGqBj/jh6vpieA4c"
+    "tJzoj3hMjBWCRmBolTUgT4oYkCdiA/IkY0CiJVmGtCe4LmOIJ4RaM8Xr5CDPTk4LIElKCa"
+    "Fk95JYBou6ttDx1CxF6GYlpUS1GWY3q78UVUo4onVqKJIa6UbKAbWjsZ72Z3UWxa1WO6e7"
+    "VLfe2zfQukOHNGqfUXLoGi4te93jmGWxu/08a4wxNEtaELVghSkjq2kjizzEhbiUnRUTUa"
+    "aWMrU4ys3hmlrKq3oQDZuhhffmwCrQipmlpYO6cbsOrL2B1qQDq1FuPvDb8Pj4yKWTw8HH"
+    "3Uf1any/9qwX7Ld62sCyISnsuIww+U1phi1rhnH0M4iKKZmUmCwaYoovPP1QhDA8/SBmDO"
+    "m9VMyJvjYtndM9f3y4vRFEnEQiaY0BGS74DzCR020fJA8/+r35Kndau071XlpBWuUmExJ6"
+    "Ls3IJqVqIQ+3TwB19tTBuwIddZBWrKJ+Sm8lgSRrtOtx2BcxiJFEe6O9R1YjTGGqC8j6Y8"
+    "xeLPuJaAOu9QQ5fKF4TUrLvRbqNY6dCXUHah52ESfSMd/QSom2Y2m1p6N3ybAKccg1mQ2q"
+    "8ptaoBeUDrrkiavwy9SUTcEJDKsktFdYQOLFRVJwItyYSnG8w2zNwom+PT15+/7th8G7tx"
+    "/6dKjTiTu88j4H7CxmK+po0Dmzi5j4jIm0uNA1Yxg0Qnsy69fRn0sP8qSgGt4p1RY7LxWB"
+    "TYsqaBVRf4B8riLqD7Rhs4HsKqplR9OK0a0loYvLvBbMVCRQg5FAmQ7ZKefRvuCLD7PqEV"
+    "TwGeJdw6cCr9DoGTZnDEoYQ5WARexM28C23aOmRa1Vt1/N9r16G0t/JxfaBZoLuQTuusBh"
+    "EYIuv5v3bEfFyGcRvjs9HQzenx4P3n04e/v+/dmH4w2dkL2VxytcXH2iRkVCGQqtDLFrTR"
+    "q+phawa6RslAOtOQeaMoUPwmLKmsJ0lSmn7kcSStkPl9FddXwZd06n9dOoX3QpuOn26uPl"
+    "g6uzd8loZNHNXHXMQlNDo37sgkHtvTuIpwRSQOsHpjVH+Bws9ZUDiP4FvrKavoKVbutL4F"
+    "oAuQ64++lyBIh1hMhn2Ue9VMPsXOEjfsQPLumWU4AwuAu6OHiDLResbMuAjgNY3P76G+BY"
+    "QPefQWrXXUBqt0mFFiZ/4COemB78HyINMSBzgmUDHU/BDGHkLCArRV/JooomkUSmCZ51E9"
+    "Hp3WGftUNYvxO2VJnQgxpTerUbu1FjlJFY11w9GVALe0kZaDOCKpQrMrH/WCEyuiroSklJ"
+    "FZyw7+AEpfQejtKbUeL2o48wHlKwxS5fC6FucLWpTv7QadqOZXOTxmXkXGkb2cW+0h3nxb"
+    "K5jCTW7bWI9omk0iEP60DH7+quYC7NGHA4m4mXQHZxdTO8/4UfrnPB4XwufhmPhmk9Zqnz"
+    "AizFvXQjIKW+fVwsDUheHpBslC8xoBwypWvYW07KadgcURkj0IuAOhBjOuDkVgnSepSHlC"
+    "OqIA3HrQYx/byyMWQZWRVExgN3sdQNUbD/lgk1JSthjz0psg3lRLwN5SSzDcVn5LzJ79Dg"
+    "2GZiQNNyEoLZiCJFrKWFtrKtZzQtN6VmBFuMfTYtw4+3rid5VZFp9Uw8rZ5lplVzqq+0uW"
+    "15K068hdgrmRJr3zPZ+/W3Zowm5ZlUJI0K0n1lDbsJuypIvvEDKWvM191l8z4/PXU8s1Z1"
+    "LJLJvLrX8wtBoVKWJ8wL3TAsb+e4zhGta+hXJTEkjrGAU4+Yo1o9yf4fwvoOINO/TT7vd2"
+    "uy67Ah1fxoTeSdSSfQRNRsmSFz195xQaq682uSuGOokw+acvixSbXH8fiNQrZc7PJjk3vB"
+    "mCNWHQhmPPK/q1MPFIvOCXIyZsOKCshsIoccGl/kwClgLwVYwRfkLsDg+NupvgY2pOXJyx"
+    "yBzwg/OTQQ6XJhW0v948Ujnlk2eIYGqQg4ULeNBTD0lT5BJnLR7pFByjdZk28yJMh5gG5P"
+    "tM0FVgrnTyOkWgV+cldqcu9u3rNCXrSzHC/aWdaLNiOzCLUvST8ulXooLScnpI30TWKeCw"
+    "HNOUTFkhHOthNdTLlhqvmUVSgjJ1klCTlVKOhvYk3XWtnThRJCkpwy0fawYBgt3GWprDoJ"
+    "IQUsP58407LJE8mrldTcOKKSgKxOv1YOiKqeJbkC9NViLW7JgPcuuYMxKaV2MYaAZDEsvZ"
+    "NRXhdCP7WdMdlJurSlMYGxiFiMtcEWfjHhPCrKMwYylCacoblnM14YUJbv6np4Ry9jyOgu"
+    "R8Q6lqghxkFupGw4pXQjeUefhYTYsNcr0tFAGN7dj9UBHOi65PFO/xFTdjOsxk9OfAQevN"
+    "XKorscl6SLoZUJwxIOWEGbnTmoOMqOcJRBpy1PB2UEFVMZMhbIWZn6Wiu7LSUtp2yH2Amg"
+    "K21hOaXY34SQLOxaW2DS+TkLpjjtUFymvZw433032AHKmlPiMAyqbDfLCKrOmOqMwo1no1"
+    "APEXNamRpkQbdtYgsRQ5sdwZGFOXdPSkKuxf0om4W/w9tRTN1xNWjbFifMX9xlk1KSLPOt"
+    "87CKLzxMvlBFoh9Ew0p1rqMUvGHnDnbcF1m4w8mOKWKjjhBuubBMxkytsZ9qjXfIWGkgHk"
+    "htD5vKJIKkcfo4hoyIQU6Ct41ETjXcdh55TG4/OYDK0RR0dNJxWOQpy9rGaFcYJ4qzPHLp"
+    "GiiPfE1hJn8dQHVaX5iuvURmuer7F4OMeHC6CY0lM2HAHM9IhZ4NAf26J5qSz7WA47PHAB"
+    "HbDy4hqd/0K6YCTGsGC/KjSYorFrkjLDKzaFinrRKulZWuQbfrloXUIVWukCuYNUkYiezx"
+    "xomQK+SJvspjz0IMBN5gIYAZuVeJHq0eGh47MzZYJjhKjBBEkfirxJINyQCF6jN0ugI1Se"
+    "95klZMymEyKSoO68DisJoOapA/CutHazJ0XbhccWOwYnf7ecbz79aETGmsoMrtKr9VaZg6"
+    "WpY/1D0l1i60+5oHE7SfgPHLz+3vtbwNrRcQQxSPHbSAZIxCkf2Sp+LtkqfZbI5lnbvKr5"
+    "vv12UHYFRSWpOSSmntmF83OLWkSsumRJVhuWfDkihW6Bna65IWSEpMmSAbRGqwQYgO/DFW"
+    "W2cB3GqEpHpJx6yQDcZ8MyTeBPl2SPCZqOgpVx89m4oD28MAsd0Y7vofAENSB7Ah02mBvp"
+    "yguWd5DiDqB40nNUH0mKxLr54quQeePpF+RG/SL/XbMTiwbKeTT5WVVMFKCpuiqK4flpcl"
+    "TrZpDT/qwkURjCTkxHBQJIP4QJxBfJDJIC4691KMofDcS0kwbCbHkSSGu39+Yoet9r0zR5"
+    "2yP0pQRwyBalHXCUllwu3ZhFO8ldqPUOfid7i8lfKiH0TDVs+MH/cYVg+RTvoou9fQe4mO"
+    "ziT75rAqvITgYmqFn45cOXqlpjB8ArDKFJyUVFrnnrVOh9GGJd0GCaFXaHPZ0EArBHmRSj"
+    "n8SVxIUSjxo0wwhoKDVkfYW2bcLakUjhvxPYPao8PxYfhpdA7CkzYf8eh6ePX5HGyOhu0A"
+    "0+Jnti9jacZEZOm4rQdJtJ/s/CATB0XaYoVwk5SsnPbdQWkXOfT49ql9L1R5YNP4Oz+P7k"
+    "Y3H69uPmX1715w5xwE3Pojvru/vSQLgH9xEyn3iL/cXI7ux8Orm3NAVHto0+2Xj/hhdDM+"
+    "Bw6ZVh/x92SNGH2kr4NI733El0Mi8pldMigyZnC28P4XD6LEeLYNg9aq0qjJGlps2PvNg4"
+    "9ugu6ealJ6mQBrYbJufyRN8ss5mJIWWT/in0ejn+ivLxA+0d+vb2/GP9ALS7IyLsx1NxrH"
+    "c6BGui5vbc/NF5SQU+dXc8JJQ+qiND0fF5RkkVc0vWJzFU2vGla42S0cdpOy0aYZwdcSbx"
+    "pHb2XDZ2R5jmYZoT5SEkZxDTVScd3dN5MTvBt1sCyerzArU2a87R65m9uPs6DfYji2yD8F"
+    "Ia9w0uye+u1W7MWDNNEID6MxuPny+XOqFUSgY6Jt5gIe+j9D4Bs53bdDh9qW9G6GB/dynJ"
+    "qxM33FvszE+cHKhSm1C7NsUmiVCzqbNz/2ZhkkxZZxSkzZxVy7eGVbQWhKUVwjCVn6aNuY"
+    "KsetctxK67iFmH45Z/nPJXdjUioVfAJPliKL7mopT1ClRFUUUxeSH5bfi5MSk0QRSZ84Xm"
+    "CeOcs7bzw9z6gjEhrTP5Sv4yAoceXrONCG7aavo1PzolzsfHeoyn4pcl7AC7eWV+MCmujO"
+    "tmaIoZghS+O3+3mE6YQU1FZ+yQZI0183R3CwB/nGmkpo0TaVGgO/hPadlFLEShLOnY7Yy9"
+    "QgC7ptK+QMKOE5SFv6rvgwpK7aO211XhvOyOyxEOVzKNqDM9VIAnPbvdino3ywKhkh3AoU"
+    "q9UFVksRMU2MGMWl18ulK2LrIPgPRWwdaMOqsz/V2Z9dPvvTpgcQeraOd03qQpmp+01lci"
+    "HbaGKXFDACUi8J3RZeL9VodVN7q4hgZI+beA7C0AnzTsTcvTaCeKqlbXL2u0Fafm7Za8UK"
+    "7ocVTDVaKX4lJSsLh9UCySLo8UXRFYi3uBl3h87aHrxlI4RTYpKY3m1xgpuZuOw0EBeUsY"
+    "ueFInQORFH6JxkInReV+j62XGxEKe8GKdsFhzkluqJYXk1qKO9SGi+mFj2wrJKLT5pOYXo"
+    "pk96iCo8WTB/fLi9EXTLSCTNsCDDBf8BJnIas4JieujEQ6aLsHNEn9eQKkpRyCeC05xvSm"
+    "elFWR3WSADasSG4SV3EHfilJiUfbj+dUm+dD/8/QAXo9HNOZhAiB/xz8ObsTa+1cb3NP8L"
+    "sXWJlaq5dkdSvpAe+KSxD8yiLjxpOylU6XztPfTdmk/YdgzL5p2qDQ201E0+cBuZ9FzrCx"
+    "0Fwp3ELwecj6PLq+vh5zdn/dOUoyfsk2+zKqjl8k53F7slNwKSzJVteyRdfc7BU7zyh+XV"
+    "sl9l2Z9B16h48mFCUrm1uuXWUn7og2jXrB86cAaUTpUTl1Jey7hPZUfHZSoEvLP4FcjUEu"
+    "8jXTrccmi7aOY71DLOus29fp6bTg9KFTzRcghICc9wPRtOgWtZJiAdi7QFeMP2wBOJiTed"
+    "Q7cPqDb36J0en777hpQhbUglEAbuAoLwoWClY2hmD7ls7Ckqq0o780s/x+nnBoFwRYmUsL"
+    "yc1HQjNODrOh703dsCCL57KwSQ3kpZUsjlrXA5fTAUkBPBZlL7kAFexhwNy7dvjjamZTVi"
+    "eDqWZ1PmGM7KoJuU2gnjTjEpjUD8DG2Hm5FKSIjGJCqxoZX67ckO479mLlRZzQdqNavo7Y"
+    "No2CzNZWE6Z7FY2LKJCbKizfIinUkcHAcwSPNVErukVKMZHTqJmto0oDYNtLRpQDTb1YDe"
+    "Zaq6rk5x21OIZKfy7WAKTzLZCcc6k0fvCc3k5N6pbCy/YH0yNAzL42/ciN/u59HBa1JQ0/"
+    "2SHUthLWZn6oxNbzo7fJKYGbwrwMsM0lpkRMvQWyqHdcNEF8OgBIRheTkhrD9bJMHAd9dk"
+    "QczdS5+Qa3E3fWNaYZ276U3LKZ2cIBJSaGZC1bJY5keqVR7hnaJXm0mZpJs69+yR3NDJmN"
+    "TOwZPd6qlh9OTJWX9QOHyS4KvbNPtfJTA50gpU+uq7wcqVV8Bu0kORpxio2tHM/BpUhqk9"
+    "Z5jyTdyymkZMSp0Wy+H79kcSXIZbZQUsweZ+fytNEOy6RU2kbVVEQdtEQbiHWpvblrcqmS"
+    "6AK9yU/SsF0IqBqZGBWaDplJclNHcNioSUtZtAkzwN2ljnBGHm4hkXU4u6IhAaJBDmlm5q"
+    "ZWnXhJAkwDa9+dfHRLe55GuupZuSPMztrKVM3Bgi2tLCpMWyiJLv3wpnJJxn3EoHKTFWM2"
+    "d/KtPxEE3HT1TH722xH/1C/aJGZGA5KEtSGgNHuZyVwdPkgqIMni5Dqtb2A1vbFyxRlWhR"
+    "X4RprLat5qxg0b3EqxXE028tbK6Bac2BNQMvC90FUDcWwFljA/jVTft+FhEwWYMpms1IYw"
+    "F9TtrHcdk+3yWiR1lwdhHXW/8jfsS/3Awv/u6A4d0VWOgOwBZYIMcl6ksfOBYpjBxA/lAh"
+    "9lTqzrKn9MFwilwHvIFH8yOgk4p0x0FzvITYBXOLPnBmW0vwt7fHx8C1wN9oOsIjcOUCh9"
+    "gMRFD332SGbPJO9NWb2M18gebC/U9cfYez9SlQXfaq7vhbn747PR0M3p8eD959OHv7/v3Z"
+    "h+PNHqjsrbzNUBdXn+hwTQzrcEIUK0G0kSr5QROC7eyqqbsBDmVTTdYb6kD7GdraE7lvwi"
+    "kvRjpvEPGk29tNKM+oih/C4yJ3XZoKTInJ6YgqQgYOxGTgIENgBaiUM3QTQlICWX/6AN3g"
+    "7zMRoxhJyGnk1p8IlHwaLJUGZCMgJ4LNpKiO1O0kjDnpgCMRlYghmyWgQybZNXMTCCyy69"
+    "CHsMUgY74Gxao2vsjUNUGKDYrSLqe63Eztr9/l3EzK916r7x1hw+Lx97ne4khIBUOTNYDt"
+    "WuKxo/kbIGJiCkZfaX7mHsKRC2NcTMFI8+dpE6hV7JNZYQUp6WJzqFkzqlxBTu8U8j9psV"
+    "eaV1/5jnoH5Ttitsi2fQXJQv1ihkvZLQa9O2h/yyRBGFcC9CXLZXAEvn4Np7GvX6mXhvmC"
+    "qEOHlDVNBzz2hswpA6ePvawzqdaalUG11aDKnVM2M/N7oUFFbymDqpJBtQnJqrgHRGKuuP"
+    "7dH8oWULZAd2BUO/NrAFHpr4elv97pawhFeqt/c7u+uqLlFNEuzQqtwpcb9EuSp2FnBu0w"
+    "k1xJTVIgLglh37RGqZafw1p+HrwJ6/BBjIhgHUqV2r4gOQmBgvzJZ4ShQ+NVdeCsCGIgVs"
+    "U/WAjqSrchdv/uRDQIogTHAy3N401qqVGti61FOeWc2xM1XJX5PCEpJTnQQCDZMszIWsam"
+    "3QgpY4zm5F1aZbz/YXlJlIm2vf/MjCk5vOMyksDatI62Rwq19ezZTWOpjAllTChjoqgxsc"
+    "bGg6u7Qj4rKlDAhCBlNWdTeC+8lu8IKTnmE0JKm+dp8wFEZcmulJiUs2gjO/Y7sjNsb9FB"
+    "LW8MY9liK++xzEqrPLN7zjPLmmRGPtNfdSq2aboC1axdaFZI98+XMdKTUpIsMm2Y6h3SNM"
+    "fbOetxKcK6NFut/Kid1jDDaLii4XVh+QOPrlOcr+J897+QcE4aKUVURiJyhj3Un2qVrG62"
+    "9VyaSouLqbRtCURnpj6nSPF0R3HXTEpJMuQb75yVqPMaGPN9+2zrZ8w76iGTDEXlIFMOMi"
+    "nARcuVZbt+eG9ptpwrLCWwjYSJhh2uLKxpOSkRbcQLUa2bqv4pSK8UrjZlAc0ISolpIz1U"
+    "BRz0dreSOkIDD6GNDG6KquBOP4/01aMynWF6Dyj/7Y7pGsS87jO0nZLpD2MiktJFTUyFdG"
+    "iU0Xr84nICeFJofT7JWZ9POOuzhV3I49Jz0h9GIir9YcfSH/71XxWPG/0="
+)

migrations/models/6_20260603000000_add_scheduled_messages.py

@@ -0,0 +1,38 @@
+"""Create scheduled_messages, which no committed migration ever created.
+
+Migrations 6–10 were generated outside the repository and never committed, and
+one of them created this table. Deployments that ran them already have it, so
+this is IF NOT EXISTS and a no-op there; fresh databases need it before
+migration 20 alters the table. Aerich cannot generate this file because its
+saved model state already includes the table: the DDL below is Tortoise's
+schema output for the model as of migration 19.
+"""
+
+from tortoise import BaseDBAsyncClient
+
+RUN_IN_TRANSACTION = True
+
+
+async def upgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        CREATE TABLE IF NOT EXISTS "scheduled_messages" (
+    "id" UUID NOT NULL PRIMARY KEY,
+    "recipient" VARCHAR(255) NOT NULL,
+    "channel" VARCHAR(20) NOT NULL,
+    "content" TEXT NOT NULL,
+    "subject" VARCHAR(255),
+    "scheduled_at" TIMESTAMPTZ NOT NULL,
+    "status" VARCHAR(20) NOT NULL DEFAULT 'pending',
+    "recurrence" VARCHAR(20) NOT NULL DEFAULT 'none',
+    "use_agent" BOOL NOT NULL DEFAULT False,
+    "error_message" TEXT,
+    "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by_id" UUID NOT NULL REFERENCES "users" ("id") ON DELETE CASCADE
+);"""
+
+
+async def downgrade(db: BaseDBAsyncClient) -> str:
+    # Existing deployments created this table in an uncommitted migration;
+    # never drop it from under them.
+    return ""

mkdocs.yml

@@ -25,3 +25,4 @@ nav:
       - Authentication: authentication.md
       - Desktop Client (proposal): desktop-client.md
       - Cowork UI plan: cowork-ui-plan.md
+      - Immich MCP (design): immich-mcp/design.md

raggr-frontend/src/App.tsx

@@ -4,6 +4,7 @@ import { AuthProvider } from "./contexts/AuthContext";
 import { ChatScreen } from "./components/ChatScreen";
 import { ConversationsPage } from "./components/ConversationsPage";
 import { LoginScreen } from "./components/LoginScreen";
+import { SourceViewer } from "./components/SourceViewer";
 import { useAuthCheck } from "./hooks/useAuthCheck";
 import catIcon from "./assets/cat.png";
 
@@ -32,6 +33,7 @@ const AppContainer = () => {
   return (
     <Routes>
       <Route path="/conversations" element={<ConversationsPage />} />
+      <Route path="/sources/:sourceId" element={<SourceViewer />} />
       {/* One ChatScreen instance serves both the home ("/") and per-conversation
           ("/c/:id") routes. Using a single catch-all route keeps that instance
           mounted across the navigate() that fires when the first message creates

raggr-frontend/src/api/artifactTypes.ts

@@ -24,6 +24,8 @@ export type ArtifactSourceRef = {
   id?: string;
   path?: string;
   url?: string;
+  as_of?: string | null;
+  stale?: boolean;
 };
 
 /** Full artifact as returned by `GET /api/artifacts/<id>`. */

raggr-frontend/src/api/conversationService.ts

@@ -4,288 +4,461 @@ import type { ArtifactRef } from "./artifactTypes";
 export type ToolInput = Record<string, string | number | boolean>;
 
 export type SSEEvent =
-  | { type: "thinking" }
-  | { type: "thinking_delta"; delta: string }
-  // v2 servers send run_id/label/input and run_id/label/status/ms/artifact;
-  // older servers send only `tool`, so everything else is optional here.
-  | {
-      type: "tool_start";
-      tool: string;
-      run_id?: string;
-      label?: string;
-      input?: ToolInput;
-    }
-  | {
-      type: "tool_end";
-      tool: string;
-      run_id?: string;
-      label?: string;
-      status?: "ok" | "error";
-      ms?: number;
-      artifact?: ArtifactRef | null;
-    }
-  | { type: "content"; delta: string }
-  | { type: "response"; message: string }
-  | { type: "error"; message: string; stalled?: boolean };
+	| { type: "thinking" }
+	| { type: "thinking_delta"; delta: string }
+	// v2 servers send run_id/label/input and run_id/label/status/ms/artifact;
+	// older servers send only `tool`, so everything else is optional here.
+	| {
+			type: "tool_start";
+			tool: string;
+			run_id?: string;
+			label?: string;
+			input?: ToolInput;
+	  }
+	| {
+			type: "tool_end";
+			tool: string;
+			run_id?: string;
+			label?: string;
+			status?: "ok" | "error";
+			ms?: number;
+			artifact?: ArtifactRef | null;
+	  }
+	| { type: "content"; delta: string }
+	| { type: "response"; message: string }
+	| { type: "error"; message: string; stalled?: boolean };
 
 export type SSEEventCallback = (event: SSEEvent) => void;
+export interface ChatRun {
+	id: string;
+	conversation_id: string;
+	status: "running" | "completed" | "failed" | "cancelled" | "interrupted";
+	cursor: number;
+	created_at: string;
+	query: string;
+	image_key?: string | null;
+}
+
+export class RunRequestError extends Error {
+	constructor(
+		message: string,
+		public status: number,
+	) {
+		super(message);
+	}
+}
 
 interface Message {
-  id: string;
-  text: string;
-  speaker: "user" | "simba" | "tool";
-  created_at: string;
-  image_key?: string | null;
-  // Tool messages only (v2 servers).
-  tool?: string;
-  label?: string;
-  artifact?: ArtifactRef | null;
+	id: string;
+	text: string;
+	speaker: "user" | "simba" | "tool";
+	created_at: string;
+	image_key?: string | null;
+	// Tool messages only (v2 servers).
+	tool?: string;
+	label?: string;
+	artifact?: ArtifactRef | null;
 }
 
 interface Conversation {
-  id: string;
-  name: string;
-  messages?: Message[];
-  created_at: string;
-  updated_at: string;
-  user_id?: string;
+	id: string;
+	name: string;
+	messages?: Message[];
+	created_at: string;
+	updated_at: string;
+	user_id?: string;
 }
 
 export type UploadResult =
-  | { kind: "image"; image_key: string }
-  | { kind: "file"; file_key: string; name: string; chars: number };
+	| { kind: "image"; image_key: string }
+	| { kind: "file"; file_key: string; name: string; chars: number };
 
 export class UploadError extends Error {}
 
 interface QueryResponse {
-  response: string;
+	response: string;
 }
 
 class ConversationService {
-  private baseUrl = "/api";
-  private conversationBaseUrl = "/api/conversation";
+	private baseUrl = "/api";
+	private conversationBaseUrl = "/api/conversation";
 
-  async sendQuery(
-    query: string,
-    conversation_id: string,
-    signal?: AbortSignal,
-  ): Promise<QueryResponse> {
-    const response = await userService.fetchWithAuth(
-      `${this.conversationBaseUrl}/query`,
-      {
-        method: "POST",
-        body: JSON.stringify({ query, conversation_id }),
-        signal,
-      },
-    );
+	async sendQuery(
+		query: string,
+		conversation_id: string,
+		signal?: AbortSignal,
+	): Promise<QueryResponse> {
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/query`,
+			{
+				method: "POST",
+				body: JSON.stringify({ query, conversation_id }),
+				signal,
+			},
+		);
 
-    if (!response.ok) {
-      throw new Error("Failed to send query");
-    }
+		if (!response.ok) {
+			throw new Error("Failed to send query");
+		}
 
-    return await response.json();
-  }
+		return await response.json();
+	}
 
-  async getMessages(): Promise<Conversation> {
-    const response = await userService.fetchWithAuth(
-      `${this.baseUrl}/messages`,
-      {
-        method: "GET",
-      },
-    );
+	async getMessages(): Promise<Conversation> {
+		const response = await userService.fetchWithAuth(
+			`${this.baseUrl}/messages`,
+			{
+				method: "GET",
+			},
+		);
 
-    if (!response.ok) {
-      throw new Error("Failed to fetch messages");
-    }
+		if (!response.ok) {
+			throw new Error("Failed to fetch messages");
+		}
 
-    return await response.json();
-  }
+		return await response.json();
+	}
 
-  async getConversation(conversationId: string): Promise<Conversation> {
-    const response = await userService.fetchWithAuth(
-      `${this.conversationBaseUrl}/${conversationId}`,
-      {
-        method: "GET",
-      },
-    );
+	async getConversation(conversationId: string): Promise<Conversation> {
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/${conversationId}`,
+			{
+				method: "GET",
+			},
+		);
 
-    if (!response.ok) {
-      throw new Error("Failed to fetch conversation");
-    }
+		if (!response.ok) {
+			throw new Error("Failed to fetch conversation");
+		}
 
-    return await response.json();
-  }
+		return await response.json();
+	}
 
-  async createConversation(): Promise<Conversation> {
-    const response = await userService.fetchWithAuth(
-      `${this.conversationBaseUrl}/`,
-      {
-        method: "POST",
-      },
-    );
+	async createConversation(): Promise<Conversation> {
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/`,
+			{
+				method: "POST",
+			},
+		);
 
-    if (!response.ok) {
-      throw new Error("Failed to create conversation");
-    }
+		if (!response.ok) {
+			throw new Error("Failed to create conversation");
+		}
 
-    return await response.json();
-  }
+		return await response.json();
+	}
 
-  async renameConversation(
-    conversationId: string,
-    name: string,
-  ): Promise<Conversation> {
-    const response = await userService.fetchWithAuth(
-      `${this.conversationBaseUrl}/${conversationId}`,
-      {
-        method: "PATCH",
-        body: JSON.stringify({ name }),
-      },
-    );
+	async renameConversation(
+		conversationId: string,
+		name: string,
+	): Promise<Conversation> {
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/${conversationId}`,
+			{
+				method: "PATCH",
+				body: JSON.stringify({ name }),
+			},
+		);
 
-    if (!response.ok) {
-      throw new Error("Failed to rename conversation");
-    }
+		if (!response.ok) {
+			throw new Error("Failed to rename conversation");
+		}
 
-    return await response.json();
-  }
+		return await response.json();
+	}
 
-  async deleteConversation(conversationId: string): Promise<void> {
-    const response = await userService.fetchWithAuth(
-      `${this.conversationBaseUrl}/${conversationId}`,
-      {
-        method: "DELETE",
-      },
-    );
+	async deleteConversation(conversationId: string): Promise<void> {
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/${conversationId}`,
+			{
+				method: "DELETE",
+			},
+		);
 
-    if (!response.ok) {
-      throw new Error("Failed to delete conversation");
-    }
-  }
+		if (!response.ok) {
+			throw new Error("Failed to delete conversation");
+		}
+	}
 
-  async getAllConversations(params?: {
-    limit?: number;
-    offset?: number;
-    search?: string;
-  }): Promise<Conversation[]> {
-    const search = new URLSearchParams();
-    if (params?.limit != null) search.set("limit", String(params.limit));
-    if (params?.offset != null) search.set("offset", String(params.offset));
-    if (params?.search) search.set("search", params.search);
-    const qs = search.toString();
+	async getAllConversations(params?: {
+		limit?: number;
+		offset?: number;
+		search?: string;
+	}): Promise<Conversation[]> {
+		const search = new URLSearchParams();
+		if (params?.limit != null) search.set("limit", String(params.limit));
+		if (params?.offset != null) search.set("offset", String(params.offset));
+		if (params?.search) search.set("search", params.search);
+		const qs = search.toString();
 
-    const response = await userService.fetchWithAuth(
-      `${this.conversationBaseUrl}/${qs ? `?${qs}` : ""}`,
-      {
-        method: "GET",
-      },
-    );
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/${qs ? `?${qs}` : ""}`,
+			{
+				method: "GET",
+			},
+		);
 
-    if (!response.ok) {
-      throw new Error("Failed to fetch conversations");
-    }
+		if (!response.ok) {
+			throw new Error("Failed to fetch conversations");
+		}
 
-    return await response.json();
-  }
+		return await response.json();
+	}
 
-  async uploadAttachment(
-    file: File,
-    conversationId: string,
-  ): Promise<UploadResult> {
-    const formData = new FormData();
-    formData.append("file", file);
-    formData.append("conversation_id", conversationId);
+	async uploadAttachment(
+		file: File,
+		conversationId: string,
+	): Promise<UploadResult> {
+		const formData = new FormData();
+		formData.append("file", file);
+		formData.append("conversation_id", conversationId);
 
-    const response = await userService.fetchWithAuth(
-      `${this.conversationBaseUrl}/upload`,
-      {
-        method: "POST",
-        body: formData,
-      },
-      { skipContentType: true },
-    );
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/upload`,
+			{
+				method: "POST",
+				body: formData,
+			},
+			{ skipContentType: true },
+		);
 
-    if (!response.ok) {
-      const data = await response.json().catch(() => ({}));
-      throw new UploadError(data.error || "Failed to upload file");
-    }
+		if (!response.ok) {
+			const data = await response.json().catch(() => ({}));
+			throw new UploadError(data.error || "Failed to upload file");
+		}
 
-    return await response.json();
-  }
+		return await response.json();
+	}
 
-  async getPresignedImageUrl(imageKey: string): Promise<string> {
-    const response = await userService.fetchWithAuth(
-      `${this.conversationBaseUrl}/image/${imageKey}`,
-    );
-    if (!response.ok) {
-      throw new Error("Failed to get image URL");
-    }
-    const data = await response.json();
-    return data.url;
-  }
+	async getPresignedImageUrl(imageKey: string): Promise<string> {
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/image/${imageKey}`,
+		);
+		if (!response.ok) {
+			throw new Error("Failed to get image URL");
+		}
+		const data = await response.json();
+		return data.url;
+	}
 
-  async streamQuery(
-    query: string,
-    conversation_id: string,
-    onEvent: SSEEventCallback,
-    signal?: AbortSignal,
-    imageKey?: string,
-    fileKey?: string,
-  ): Promise<void> {
-    const body: Record<string, string> = { query, conversation_id };
-    if (imageKey) {
-      body.image_key = imageKey;
-    }
-    if (fileKey) {
-      body.file_key = fileKey;
-    }
+	async createRun(
+		body: Record<string, string>,
+		signal?: AbortSignal,
+	): Promise<ChatRun> {
+		// Retry ambiguous transport failures with the SAME idempotency key.
+		for (let attempt = 0; ; attempt++) {
+			try {
+				const response = await userService.fetchWithAuth(
+					`${this.conversationBaseUrl}/runs`,
+					{
+						method: "POST",
+						body: JSON.stringify(body),
+						signal,
+					},
+				);
+				if (!response.ok) {
+					const data = await response.json().catch(() => ({}));
+					throw new RunRequestError(
+						data.error || "Could not start generation",
+						response.status,
+					);
+				}
+				return await response.json();
+			} catch (error) {
+				if (
+					signal?.aborted ||
+					error instanceof RunRequestError ||
+					attempt >= 2 ||
+					(error instanceof Error && error.message.includes("Session expired"))
+				)
+					throw error;
+				await new Promise((resolve) =>
+					setTimeout(resolve, 500 * (attempt + 1)),
+				);
+			}
+		}
+	}
 
-    const response = await userService.fetchWithAuth(
-      `${this.conversationBaseUrl}/stream-query`,
-      {
-        method: "POST",
-        body: JSON.stringify(body),
-        signal,
-      },
-    );
+	async discoverRun(conversationId: string): Promise<ChatRun | null> {
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/${conversationId}/runs`,
+		);
+		if (!response.ok) throw new Error("Could not discover active generation");
+		return (await response.json()).run;
+	}
 
-    if (!response.ok) {
-      throw new Error("Failed to stream query");
-    }
+	async cancelRun(run: ChatRun): Promise<void> {
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/${run.conversation_id}/runs/${run.id}/cancel`,
+			{ method: "POST" },
+		);
+		if (!response.ok)
+			throw new Error("Could not stop generation. Try Stop again.");
+	}
 
-    await this._readSSEStream(response, onEvent);
-  }
+	async resumeRun(
+		run: ChatRun,
+		onEvent: SSEEventCallback,
+		signal: AbortSignal,
+		onConnection?: (reconnecting: boolean) => void,
+	): Promise<void> {
+		signal = AbortSignal.any([signal, AbortSignal.timeout(35 * 60 * 1000)]);
+		let cursor = 0;
+		let failures = 0;
+		const deadline = Date.now() + 35 * 60 * 1000;
+		while (!signal.aborted && Date.now() < deadline) {
+			try {
+				const response = await userService.fetchWithAuth(
+					`${this.conversationBaseUrl}/${run.conversation_id}/runs/${run.id}/events?cursor=${cursor}`,
+					{ signal },
+				);
+				if (!response.ok)
+					throw new RunRequestError(
+						"Could not resume generation",
+						response.status,
+					);
+				onConnection?.(false);
+				const done = await this._readSSEStream(
+					response,
+					onEvent,
+					(id) => {
+						cursor = id;
+						failures = 0;
+					},
+					cursor,
+				);
+				if (done) return;
+				throw new Error("Connection ended before generation finished");
+			} catch (error) {
+				if (
+					signal.aborted ||
+					error instanceof RunRequestError ||
+					(error instanceof Error && error.message.includes("Session expired"))
+				)
+					throw error;
+				onConnection?.(true);
+				if (++failures >= 8)
+					throw new Error(
+						"Connection lost. Reload this conversation to resume.",
+					);
+				await new Promise<void>((resolve, reject) => {
+					const aborted = () => {
+						clearTimeout(timer);
+						reject(new DOMException("Detached", "AbortError"));
+					};
+					const timer = setTimeout(
+						() => {
+							signal.removeEventListener("abort", aborted);
+							resolve();
+						},
+						Math.min(500 * 2 ** failures, 5000),
+					);
+					signal.addEventListener("abort", aborted, { once: true });
+				});
+			}
+		}
+		if (signal.aborted) throw new DOMException("Detached", "AbortError");
+		throw new Error(
+			"Connection timed out. Reload this conversation to resume.",
+		);
+	}
 
-  private async _readSSEStream(
-    response: Response,
-    onEvent: SSEEventCallback,
-  ): Promise<void> {
-    const reader = response.body!.getReader();
-    const decoder = new TextDecoder();
-    let buffer = "";
+	async streamQuery(
+		query: string,
+		conversation_id: string,
+		onEvent: SSEEventCallback,
+		signal?: AbortSignal,
+		imageKey?: string,
+		fileKey?: string,
+	): Promise<void> {
+		const body: Record<string, string> = { query, conversation_id };
+		if (imageKey) {
+			body.image_key = imageKey;
+		}
+		if (fileKey) {
+			body.file_key = fileKey;
+		}
 
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) break;
+		const response = await userService.fetchWithAuth(
+			`${this.conversationBaseUrl}/stream-query`,
+			{
+				method: "POST",
+				body: JSON.stringify(body),
+				signal,
+			},
+		);
 
-      buffer += decoder.decode(value, { stream: true });
-      const parts = buffer.split("\n\n");
-      buffer = parts.pop() ?? "";
+		if (!response.ok) {
+			throw new Error("Failed to stream query");
+		}
 
-      for (const part of parts) {
-        const line = part.trim();
-        if (!line.startsWith("data: ")) continue;
-        const data = line.slice(6);
-        if (data === "[DONE]") return;
-        try {
-          const event = JSON.parse(data) as SSEEvent;
-          onEvent(event);
-        } catch {
-          // ignore malformed events
-        }
-      }
-    }
-  }
+		await this._readSSEStream(response, onEvent);
+	}
+
+	private async _readSSEStream(
+		response: Response,
+		onEvent: SSEEventCallback,
+		onCursor?: (cursor: number) => void,
+		initialCursor = 0,
+	): Promise<boolean> {
+		if (!response.body) throw new Error("Missing event stream");
+		const reader = response.body.getReader();
+		const decoder = new TextDecoder();
+		let buffer = "";
+		let cursor = initialCursor;
+		const frame = (part: string): boolean => {
+			const lines = part.split("\n");
+			const data = lines
+				.filter((line) => line.startsWith("data:"))
+				.map((line) => line.slice(5).trimStart())
+				.join("\n");
+			const rawId = lines
+				.find((line) => line.startsWith("id:"))
+				?.slice(3)
+				.trim();
+			const id = rawId ? Number(rawId) : undefined;
+			if (!data || (id !== undefined && id <= cursor)) return false;
+			if (data === "[DONE]") return true;
+			// Invalid payloads fail the connection so a durable event is never silently skipped.
+			const event = JSON.parse(data) as SSEEvent;
+			onEvent(event);
+			if (id !== undefined && Number.isSafeInteger(id) && id > 0) {
+				cursor = id;
+				onCursor?.(id);
+			}
+			return false;
+		};
+		try {
+			while (true) {
+				let timer: ReturnType<typeof setTimeout> | undefined;
+				const { done, value } = await Promise.race([
+					reader.read(),
+					new Promise<never>((_, reject) => {
+						timer = setTimeout(
+							() => reject(new Error("Event stream went silent")),
+							25000,
+						);
+					}),
+				]).finally(() => {
+					if (timer !== undefined) clearTimeout(timer);
+				});
+				if (done) return false;
+				buffer += decoder.decode(value, { stream: true });
+				// Normalize complete CRLF pairs, including a pair split across chunks.
+				buffer = buffer.replace(/\r\n/g, "\n");
+				let boundary: number;
+				while ((boundary = buffer.indexOf("\n\n")) !== -1) {
+					const part = buffer.slice(0, boundary);
+					buffer = buffer.slice(boundary + 2);
+					if (frame(part)) return true;
+				}
+			}
+		} finally {
+			await reader.cancel().catch(() => {});
+			reader.releaseLock();
+		}
+	}
 }
 
 export const conversationService = new ConversationService();

raggr-frontend/src/api/historyService.ts

@@ -0,0 +1,56 @@
+import { userService } from "./userService";
+
+export type HistoryConversation = {
+	id: string;
+	name: string;
+	pinned: boolean;
+	snippet?: string;
+	match_message_id?: string | null;
+};
+export type SavedMemory = { id: string; content: string; updated_at: string };
+
+async function request<T>(path: string, options?: RequestInit): Promise<T> {
+	const response = await userService.fetchWithAuth(
+		`/api/history${path}`,
+		options,
+	);
+	if (!response.ok)
+		throw new Error("Unable to update or load history. Please try again.");
+	return response.status === 204 ? (undefined as T) : response.json();
+}
+
+export const historyService = {
+	list: ({ limit = 20, offset = 0, search = "" } = {}) =>
+		request<HistoryConversation[]>(
+			`/conversations?${new URLSearchParams({ limit: String(limit), offset: String(offset), search })}`,
+		),
+	pin: (id: string, pinned: boolean) =>
+		request(`/conversations/${encodeURIComponent(id)}/pin`, {
+			method: "PATCH",
+			body: JSON.stringify({ pinned }),
+		}),
+	async export(id: string, format: "json" | "markdown") {
+		const response = await userService.fetchWithAuth(
+			`/api/history/conversations/${encodeURIComponent(id)}/export?format=${format}`,
+		);
+		if (!response.ok)
+			throw new Error("Unable to export this conversation. Please try again.");
+		const url = URL.createObjectURL(await response.blob());
+		const link = document.createElement("a");
+		link.href = url;
+		link.download = `conversation-${id}.${format === "json" ? "json" : "md"}`;
+		document.body.appendChild(link);
+		link.click();
+		link.remove();
+		setTimeout(() => URL.revokeObjectURL(url), 1000);
+	},
+	memories: (offset = 0) =>
+		request<SavedMemory[]>(`/memories?limit=100&offset=${offset}`),
+	correctMemory: (id: string, content: string) =>
+		request<SavedMemory>(`/memories/${encodeURIComponent(id)}`, {
+			method: "PATCH",
+			body: JSON.stringify({ content }),
+		}),
+	forgetMemory: (id: string) =>
+		request<void>(`/memories/${encodeURIComponent(id)}`, { method: "DELETE" }),
+};

raggr-frontend/src/api/scheduledMessageService.ts

@@ -1,13 +1,18 @@
 import { userService } from "./userService";
 
+export type ScheduledMessageStatus = "pending" | "processing" | "sent" | "failed" | "uncertain" | "cancelled";
+export type DeliveryStatus = ScheduledMessageStatus | "sending";
+
 export interface ScheduledMessage {
   id: string;
+  series_id: string;
+  retryable: boolean;
   recipient: string;
   channel: "imessage" | "email";
   content: string;
   subject: string | null;
   scheduled_at: string;
-  status: "pending" | "sent" | "failed" | "cancelled";
+  status: ScheduledMessageStatus;
   recurrence: "none" | "daily" | "weekly" | "monthly";
   use_agent: boolean;
   error_message: string | null;
@@ -15,6 +20,23 @@ export interface ScheduledMessage {
   updated_at: string;
 }
 
+export interface DeliveryAttempt {
+  id: string;
+  status: DeliveryStatus;
+  error: string | null;
+  started_at: string;
+  finished_at: string | null;
+}
+
+export interface ScheduledDelivery {
+  id: string;
+  run_id: string;
+  status: DeliveryStatus;
+  error: string | null;
+  retryable: boolean;
+  attempts: DeliveryAttempt[];
+}
+
 export interface CreateScheduledMessage {
   recipient: string;
   channel: "imessage" | "email";
@@ -25,12 +47,27 @@ export interface CreateScheduledMessage {
   use_agent?: boolean;
 }
 
+export function canRetryScheduledMessage(message: Pick<ScheduledMessage, "status" | "retryable">): boolean {
+  return message.status === "failed" && message.retryable === true;
+}
+
+async function checkResponse(response: Response, fallback: string): Promise<void> {
+  if (response.ok) return;
+  const payload = await response.json().catch(() => null);
+  const detail = payload?.error || payload?.error_message;
+  throw new Error(typeof detail === "string" ? detail : `${fallback}${payload?.status ? ` (${payload.status})` : ""}`);
+}
+
 class ScheduledMessageService {
   private baseUrl = "/api/scheduled-messages";
 
+  private messageUrl(id: string): string {
+    return `${this.baseUrl}/${encodeURIComponent(id)}`;
+  }
+
   async list(): Promise<ScheduledMessage[]> {
     const response = await userService.fetchWithAuth(`${this.baseUrl}/`);
-    if (!response.ok) throw new Error("Failed to list scheduled messages");
+    await checkResponse(response, "Failed to list scheduled messages");
     return response.json();
   }
 
@@ -39,33 +76,42 @@ class ScheduledMessageService {
       method: "POST",
       body: JSON.stringify(data),
     });
-    if (!response.ok) {
-      const err = await response.json();
-      throw new Error(err.error ?? "Failed to create scheduled message");
-    }
+    await checkResponse(response, "Failed to create scheduled message");
     return response.json();
   }
 
-  async update(id: string, data: Partial<CreateScheduledMessage> & { status?: string }): Promise<ScheduledMessage> {
-    const response = await userService.fetchWithAuth(`${this.baseUrl}/${id}`, {
+  async update(id: string, data: Partial<CreateScheduledMessage> & { status?: "cancelled" }): Promise<ScheduledMessage> {
+    // Runtime guard also protects callers using untyped data: retry has its own endpoint.
+    if (data.status !== undefined && data.status !== "cancelled") {
+      throw new Error("Status changes require the explicit retry endpoint; only cancellation is allowed here");
+    }
+    const response = await userService.fetchWithAuth(this.messageUrl(id), {
       method: "PUT",
       body: JSON.stringify(data),
     });
-    if (!response.ok) {
-      const err = await response.json();
-      throw new Error(err.error ?? "Failed to update scheduled message");
-    }
+    await checkResponse(response, "Failed to update scheduled message");
     return response.json();
   }
 
   async remove(id: string): Promise<void> {
-    const response = await userService.fetchWithAuth(`${this.baseUrl}/${id}`, {
+    const response = await userService.fetchWithAuth(this.messageUrl(id), {
       method: "DELETE",
     });
-    if (!response.ok) {
-      const err = await response.json();
-      throw new Error(err.error ?? "Failed to delete scheduled message");
-    }
+    await checkResponse(response, "Failed to delete scheduled message");
+  }
+
+  async deliveries(id: string): Promise<ScheduledDelivery[]> {
+    const response = await userService.fetchWithAuth(`${this.messageUrl(id)}/deliveries`);
+    await checkResponse(response, "Failed to load delivery history");
+    return response.json();
+  }
+
+  async retry(id: string): Promise<ScheduledMessage> {
+    const response = await userService.fetchWithAuth(`${this.messageUrl(id)}/retry`, {
+      method: "POST",
+    });
+    await checkResponse(response, "Failed to retry scheduled message");
+    return response.json();
   }
 }
 

raggr-frontend/src/components/ChatScreen.tsx

@@ -11,7 +11,10 @@ import { SidebarContent } from "./SidebarContent";
 import { AdminPanel } from "./AdminPanel";
 import { ScheduledMessagesPanel } from "./ScheduledMessagesPanel";
 import { ArtifactPanel } from "./artifacts/ArtifactPanel";
-import { AUTO_OPEN_MIN_WIDTH, useArtifactPanel } from "./artifacts/useArtifactPanel";
+import {
+	AUTO_OPEN_MIN_WIDTH,
+	useArtifactPanel,
+} from "./artifacts/useArtifactPanel";
 import { cn } from "../lib/utils";
 import { useConversations } from "../hooks/useConversations";
 import { useChat, type ToolMessage } from "../hooks/useChat";
@@ -19,435 +22,465 @@ import type { ArtifactRef } from "../api/artifactTypes";
 import catIcon from "../assets/cat.png";
 
 type ChatScreenProps = {
-  setAuthenticated: (isAuth: boolean) => void;
-  isAdmin: boolean;
+	setAuthenticated: (isAuth: boolean) => void;
+	isAdmin: boolean;
 };
 
 export const ChatScreen = ({ setAuthenticated, isAdmin }: ChatScreenProps) => {
-  const navigate = useNavigate();
-  const [query, setQuery] = useState("");
-  const [simbaMode, setSimbaMode] = useState(false);
-  const [showConversations, setShowConversations] = useState(false);
-  const [sidebarCollapsed, setSidebarCollapsed] = useState(false);
-  const [showAdminPanel, setShowAdminPanel] = useState(false);
-  const [showScheduler, setShowScheduler] = useState(false);
+	const navigate = useNavigate();
+	const [query, setQuery] = useState("");
+	const [simbaMode, setSimbaMode] = useState(false);
+	const [showConversations, setShowConversations] = useState(false);
+	const [sidebarCollapsed, setSidebarCollapsed] = useState(false);
+	const [showAdminPanel, setShowAdminPanel] = useState(false);
+	const [showScheduler, setShowScheduler] = useState(false);
 
-  const messagesEndRef = useRef<HTMLDivElement>(null);
-  const isLoadingRef = useRef(false);
+	const messagesEndRef = useRef<HTMLDivElement>(null);
+	const isLoadingRef = useRef(false);
 
-  const scrollToBottom = useCallback(() => {
-    requestAnimationFrame(() => {
-      messagesEndRef.current?.scrollIntoView({
-        behavior: isLoadingRef.current ? "instant" : "smooth",
-      });
-    });
-  }, []);
+	const scrollToBottom = useCallback(() => {
+		requestAnimationFrame(() => {
+			messagesEndRef.current?.scrollIntoView({
+				behavior: isLoadingRef.current ? "instant" : "smooth",
+			});
+		});
+	}, []);
 
-  const {
-    conversations,
-    selectedConversation,
-    createConversation,
-    renameConversation,
-    deleteConversation,
-    refreshConversations,
-  } = useConversations();
+	const {
+		conversations,
+		selectedConversation,
+		createConversation,
+		renameConversation,
+		deleteConversation,
+		refreshConversations,
+		pinConversation,
+		exportConversation,
+	} = useConversations();
 
-  const onSessionExpired = useCallback(() => setAuthenticated(false), [setAuthenticated]);
+	const onSessionExpired = useCallback(
+		() => setAuthenticated(false),
+		[setAuthenticated],
+	);
 
-  const conversationId = useMatch("/c/:conversationId")?.params.conversationId;
-  const conversationIdRef = useRef(conversationId);
-  conversationIdRef.current = conversationId;
+	const conversationId = useMatch("/c/:conversationId")?.params.conversationId;
+	const conversationIdRef = useRef(conversationId);
+	conversationIdRef.current = conversationId;
 
-  const artifactPanel = useArtifactPanel(conversationId);
-  // Opens an artifact in the panel; the timeline's "Open" chip calls this.
-  const openArtifact = artifactPanel.open;
+	const artifactPanel = useArtifactPanel(conversationId);
+	// Opens an artifact in the panel; the timeline's "Open" chip calls this.
+	const openArtifact = artifactPanel.open;
 
-  // Auto-open on wide windows; newest wins. Narrower windows rely on the
-  // timeline chip or the header's Artifacts button.
-  const handleArtifact = useCallback(
-    (ref: ArtifactRef, turnConversationId: string) => {
-      if (turnConversationId !== conversationIdRef.current) return;
-      if (window.innerWidth < AUTO_OPEN_MIN_WIDTH) return;
-      openArtifact(ref, { auto: true });
-    },
-    [openArtifact],
-  );
+	// Auto-open on wide windows; newest wins. Narrower windows rely on the
+	// timeline chip or the header's Artifacts button.
+	const handleArtifact = useCallback(
+		(ref: ArtifactRef, turnConversationId: string) => {
+			if (turnConversationId !== conversationIdRef.current) return;
+			if (window.innerWidth < AUTO_OPEN_MIN_WIDTH) return;
+			openArtifact(ref, { auto: true });
+		},
+		[openArtifact],
+	);
 
-  const toggleArtifacts = useCallback(() => {
-    const p = artifactPanel;
-    if (p.isOpen && (p.view === "library" || !p.active)) p.collapse();
-    else p.showLibrary();
-  }, [artifactPanel]);
+	const toggleArtifacts = useCallback(() => {
+		const p = artifactPanel;
+		if (p.isOpen && (p.view === "library" || !p.active)) p.collapse();
+		else p.showLibrary();
+	}, [artifactPanel]);
 
-  const {
-    messages,
-    isLoading,
-    streaming,
-    thinking,
-    messagesLoading,
-    pendingAttachment,
-    attachFile,
-    clearAttachment,
-    attachmentError,
-    clearAttachmentError,
-    sendMessage,
-    stopGeneration,
-    error,
-    clearError,
-    stalled,
-    retryStalled,
-    thinkingText,
-    thinkingDurationMs,
-  } = useChat({
-    createConversation,
-    refreshConversations,
-    onSessionExpired,
-    scrollToBottom,
-    onArtifact: handleArtifact,
-  });
+	const {
+		messages,
+		isLoading,
+		streaming,
+		reconnecting,
+		thinking,
+		messagesLoading,
+		pendingAttachment,
+		attachFile,
+		clearAttachment,
+		attachmentError,
+		clearAttachmentError,
+		sendMessage,
+		stopGeneration,
+		error,
+		clearError,
+		stalled,
+		retryStalled,
+		thinkingText,
+		thinkingDurationMs,
+	} = useChat({
+		createConversation,
+		refreshConversations,
+		onSessionExpired,
+		scrollToBottom,
+		onArtifact: handleArtifact,
+	});
 
-  // Keep ref in sync for scrollToBottom behavior
-  isLoadingRef.current = isLoading;
+	// Keep ref in sync for scrollToBottom behavior
+	isLoadingRef.current = isLoading;
 
-  const handleSelectConversation = useCallback(
-    (conversation: { title: string; id: string }) => {
-      setShowConversations(false);
-      navigate(`/c/${conversation.id}`);
-    },
-    [navigate],
-  );
+	const handleSelectConversation = useCallback(
+		(conversation: { title: string; id: string }) => {
+			setShowConversations(false);
+			navigate(`/c/${conversation.id}`);
+		},
+		[navigate],
+	);
 
-  const handleCreateNewConversation = useCallback(() => {
-    setShowConversations(false);
-    navigate("/");
-  }, [navigate]);
+	const handleCreateNewConversation = useCallback(() => {
+		setShowConversations(false);
+		navigate("/");
+	}, [navigate]);
 
-  const handleQuestionSubmit = useCallback(() => {
-    sendMessage(query, simbaMode);
-    setQuery("");
-  }, [query, simbaMode, sendMessage]);
+	const handleQuestionSubmit = useCallback(() => {
+		sendMessage(query, simbaMode);
+		setQuery("");
+	}, [query, simbaMode, sendMessage]);
 
-  const handleQueryChange = useCallback((event: React.ChangeEvent<HTMLTextAreaElement>) => {
-    setQuery(event.target.value);
-  }, []);
+	const handleQueryChange = useCallback(
+		(event: React.ChangeEvent<HTMLTextAreaElement>) => {
+			setQuery(event.target.value);
+		},
+		[],
+	);
 
-  const handleKeyDown = useCallback((event: React.KeyboardEvent<HTMLTextAreaElement>) => {
-    if (event.key === "Enter" && !event.shiftKey) {
-      event.preventDefault();
-      handleQuestionSubmit();
-    }
-  }, [handleQuestionSubmit]);
+	const handleKeyDown = useCallback(
+		(event: React.KeyboardEvent<HTMLTextAreaElement>) => {
+			if (event.key === "Enter" && !event.shiftKey) {
+				event.preventDefault();
+				handleQuestionSubmit();
+			}
+		},
+		[handleQuestionSubmit],
+	);
 
-  const handleToggleSimbaMode = useCallback(() => setSimbaMode((v) => !v), []);
+	const handleToggleSimbaMode = useCallback(() => setSimbaMode((v) => !v), []);
 
-  const handleDeleteConversation = useCallback(
-    (id: string) => deleteConversation(id),
-    [deleteConversation],
-  );
+	const handleDeleteConversation = useCallback(
+		(id: string) => deleteConversation(id),
+		[deleteConversation],
+	);
 
-  const handleLogout = () => {
-    localStorage.removeItem("access_token");
-    localStorage.removeItem("refresh_token");
-    setAuthenticated(false);
-  };
+	const handleLogout = () => {
+		localStorage.removeItem("access_token");
+		localStorage.removeItem("refresh_token");
+		setAuthenticated(false);
+	};
 
-  const sidebarProps = {
-    conversations,
-    onCreateNewConversation: handleCreateNewConversation,
-    onSelectConversation: handleSelectConversation,
-    onRenameConversation: renameConversation,
-    onDeleteConversation: handleDeleteConversation,
-    selectedId: selectedConversation?.id,
-    isAdmin,
-  };
+	const sidebarProps = {
+		conversations,
+		onCreateNewConversation: handleCreateNewConversation,
+		onSelectConversation: handleSelectConversation,
+		onRenameConversation: renameConversation,
+		onDeleteConversation: handleDeleteConversation,
+		onPinConversation: pinConversation,
+		onExportConversation: exportConversation,
+		selectedId: selectedConversation?.id,
+		isAdmin,
+	};
 
-  return (
-    <div className="h-screen h-[100dvh] flex flex-row bg-cream overflow-hidden">
-      {/* Desktop Sidebar */}
-      <aside
-        className={cn(
-          "hidden md:flex md:flex-col",
-          "bg-sidebar-bg transition-all duration-300 ease-in-out",
-          sidebarCollapsed ? "w-[56px]" : "w-64",
-        )}
-      >
-        {sidebarCollapsed ? (
-          <div className="flex flex-col items-center py-4 gap-4 h-full">
-            <button
-              onClick={() => setSidebarCollapsed(false)}
-              className="w-9 h-9 rounded-xl flex items-center justify-center text-cream/50 hover:text-cream hover:bg-white/10 transition-all cursor-pointer"
-            >
-              <PanelLeftOpen size={18} />
-            </button>
-            <img
-              src={catIcon}
-              alt="Simba"
-              className="w-12 h-12 opacity-70 mt-1"
-            />
-          </div>
-        ) : (
-          <SidebarContent
-            {...sidebarProps}
-            onShowAdmin={() => setShowAdminPanel(true)}
-            onShowScheduler={() => setShowScheduler(true)}
-            onLogout={handleLogout}
-            onCollapse={() => setSidebarCollapsed(true)}
-          />
-        )}
-      </aside>
+	return (
+		<div className="h-screen h-[100dvh] flex flex-row bg-cream overflow-hidden">
+			{/* Desktop Sidebar */}
+			<aside
+				className={cn(
+					"hidden md:flex md:flex-col",
+					"bg-sidebar-bg transition-all duration-300 ease-in-out",
+					sidebarCollapsed ? "w-[56px]" : "w-64",
+				)}
+			>
+				{sidebarCollapsed ? (
+					<div className="flex flex-col items-center py-4 gap-4 h-full">
+						<button
+							onClick={() => setSidebarCollapsed(false)}
+							className="w-9 h-9 rounded-xl flex items-center justify-center text-cream/50 hover:text-cream hover:bg-white/10 transition-all cursor-pointer"
+						>
+							<PanelLeftOpen size={18} />
+						</button>
+						<img
+							src={catIcon}
+							alt="Simba"
+							className="w-12 h-12 opacity-70 mt-1"
+						/>
+					</div>
+				) : (
+					<SidebarContent
+						{...sidebarProps}
+						onShowAdmin={() => setShowAdminPanel(true)}
+						onShowScheduler={() => setShowScheduler(true)}
+						onLogout={handleLogout}
+						onCollapse={() => setSidebarCollapsed(true)}
+					/>
+				)}
+			</aside>
 
-      {/* Mobile drawer */}
-      {showConversations && (
-        <div className="md:hidden fixed inset-0 z-50 flex">
-          <aside className="w-72 max-w-[80%] h-full bg-sidebar-bg shadow-xl animate-drawer-in">
-            <SidebarContent
-              {...sidebarProps}
-              onShowAdmin={() => {
-                setShowConversations(false);
-                setShowAdminPanel(true);
-              }}
-              onShowScheduler={() => {
-                setShowConversations(false);
-                setShowScheduler(true);
-              }}
-              onLogout={handleLogout}
-              onCloseDrawer={() => setShowConversations(false)}
-            />
-          </aside>
-          <div
-            className="flex-1 bg-charcoal/40 backdrop-blur-sm"
-            onClick={() => setShowConversations(false)}
-          />
-        </div>
-      )}
+			{/* Mobile drawer */}
+			{showConversations && (
+				<div className="md:hidden fixed inset-0 z-50 flex">
+					<aside className="w-72 max-w-[80%] h-full bg-sidebar-bg shadow-xl animate-drawer-in">
+						<SidebarContent
+							{...sidebarProps}
+							onShowAdmin={() => {
+								setShowConversations(false);
+								setShowAdminPanel(true);
+							}}
+							onShowScheduler={() => {
+								setShowConversations(false);
+								setShowScheduler(true);
+							}}
+							onLogout={handleLogout}
+							onCloseDrawer={() => setShowConversations(false)}
+						/>
+					</aside>
+					<div
+						className="flex-1 bg-charcoal/40 backdrop-blur-sm"
+						onClick={() => setShowConversations(false)}
+					/>
+				</div>
+			)}
 
-      {showAdminPanel && <AdminPanel onClose={() => setShowAdminPanel(false)} />}
-      {showScheduler && <ScheduledMessagesPanel onClose={() => setShowScheduler(false)} />}
+			{showAdminPanel && (
+				<AdminPanel onClose={() => setShowAdminPanel(false)} />
+			)}
+			{showScheduler && (
+				<ScheduledMessagesPanel onClose={() => setShowScheduler(false)} />
+			)}
 
-      <div className="relative flex-1 flex flex-col h-full overflow-hidden min-w-0">
-        <DropOverlay onDrop={attachFile} />
-        <header className="md:hidden flex items-center justify-between px-4 py-3 bg-warm-white border-b border-sand-light/60">
-          <div className="flex items-center gap-2">
-            <img src={catIcon} alt="Simba" className="w-12 h-12" />
-            <h1
-              className="text-base font-bold text-charcoal"
-              style={{ fontFamily: "var(--font-display)" }}
-            >
-              asksimba
-            </h1>
-          </div>
-          <div className="flex items-center gap-1">
-            {conversationId && (
-              <button
-                className="w-8 h-8 rounded-xl flex items-center justify-center text-warm-gray hover:text-charcoal hover:bg-cream-dark transition-all cursor-pointer"
-                onClick={toggleArtifacts}
-                aria-label="Artifacts"
-              >
-                <Library size={16} />
-              </button>
-            )}
-            <button
-              className="w-8 h-8 rounded-xl flex items-center justify-center text-warm-gray hover:text-charcoal hover:bg-cream-dark transition-all cursor-pointer"
-              onClick={() => setShowConversations(true)}
-              aria-label="Open menu"
-            >
-              <Menu size={16} />
-            </button>
-          </div>
-        </header>
+			<div className="relative flex-1 flex flex-col h-full overflow-hidden min-w-0">
+				<DropOverlay onDrop={attachFile} />
+				<header className="md:hidden flex items-center justify-between px-4 py-3 bg-warm-white border-b border-sand-light/60">
+					<div className="flex items-center gap-2">
+						<img src={catIcon} alt="Simba" className="w-12 h-12" />
+						<h1
+							className="text-base font-bold text-charcoal"
+							style={{ fontFamily: "var(--font-display)" }}
+						>
+							asksimba
+						</h1>
+					</div>
+					<div className="flex items-center gap-1">
+						{conversationId && (
+							<button
+								className="w-8 h-8 rounded-xl flex items-center justify-center text-warm-gray hover:text-charcoal hover:bg-cream-dark transition-all cursor-pointer"
+								onClick={toggleArtifacts}
+								aria-label="Artifacts"
+							>
+								<Library size={16} />
+							</button>
+						)}
+						<button
+							className="w-8 h-8 rounded-xl flex items-center justify-center text-warm-gray hover:text-charcoal hover:bg-cream-dark transition-all cursor-pointer"
+							onClick={() => setShowConversations(true)}
+							aria-label="Open menu"
+						>
+							<Menu size={16} />
+						</button>
+					</div>
+				</header>
 
-        {conversationId && (
-          <header className="hidden md:flex items-center justify-between gap-3 px-4 py-2 border-b border-sand-light/40">
-            <h2 className="min-w-0 truncate text-sm font-semibold text-warm-gray">
-              {selectedConversation?.title}
-            </h2>
-            <button
-              onClick={toggleArtifacts}
-              aria-pressed={artifactPanel.isOpen}
-              className={cn(
-                "inline-flex shrink-0 items-center gap-1.5 rounded-xl px-3 py-1.5 text-xs font-semibold transition-all cursor-pointer",
-                artifactPanel.isOpen
-                  ? "bg-leaf-pale text-leaf-dark"
-                  : "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
-              )}
-            >
-              <Library size={14} />
-              Artifacts
-              {artifactPanel.tabs.length > 0 && (
-                <span className="rounded-full bg-leaf/15 px-1.5 text-[10px] text-leaf-dark">
-                  {artifactPanel.tabs.length}
-                </span>
-              )}
-            </button>
-          </header>
-        )}
+				{conversationId && (
+					<header className="hidden md:flex items-center justify-between gap-3 px-4 py-2 border-b border-sand-light/40">
+						<h2 className="min-w-0 truncate text-sm font-semibold text-warm-gray">
+							{selectedConversation?.title}
+						</h2>
+						<button
+							onClick={toggleArtifacts}
+							aria-pressed={artifactPanel.isOpen}
+							className={cn(
+								"inline-flex shrink-0 items-center gap-1.5 rounded-xl px-3 py-1.5 text-xs font-semibold transition-all cursor-pointer",
+								artifactPanel.isOpen
+									? "bg-leaf-pale text-leaf-dark"
+									: "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
+							)}
+						>
+							<Library size={14} />
+							Artifacts
+							{artifactPanel.tabs.length > 0 && (
+								<span className="rounded-full bg-leaf/15 px-1.5 text-[10px] text-leaf-dark">
+									{artifactPanel.tabs.length}
+								</span>
+							)}
+						</button>
+					</header>
+				)}
 
-        {messages.length === 0 && !messagesLoading ? (
-          <div className="flex-1 flex flex-col items-center justify-center px-4 gap-6">
-            <div className="relative">
-              <div className="absolute -inset-6 bg-amber-soft/20 rounded-full blur-3xl" />
-              <img src={catIcon} alt="Simba" className="relative w-36 h-36" />
-            </div>
-            <h1
-              className="text-2xl font-bold text-charcoal"
-              style={{ fontFamily: "var(--font-display)" }}
-            >
-              Ask me anything
-            </h1>
-            <div className="w-full max-w-2xl">
-              <MessageInput
-                query={query}
-                handleQueryChange={handleQueryChange}
-                handleKeyDown={handleKeyDown}
-                handleQuestionSubmit={handleQuestionSubmit}
-                simbaMode={simbaMode}
-                onToggleSimbaMode={handleToggleSimbaMode}
-                isLoading={isLoading}
-                onStop={stopGeneration}
-                pendingAttachment={pendingAttachment}
-                onAttach={attachFile}
-                onClearAttachment={clearAttachment}
-                attachmentError={attachmentError}
-                onDismissAttachmentError={clearAttachmentError}
-                autoFocus
-              />
-            </div>
-          </div>
-        ) : (
-          <>
-            <div className="flex-1 overflow-y-auto px-4 py-6">
-              <div className="max-w-3xl mx-auto flex flex-col gap-6">
-                {messages.map((msg, index) => {
-                  if (msg.speaker === "tool") {
-                    // Consecutive tool calls render as one timeline block,
-                    // emitted at the first row of the run.
-                    if (index > 0 && messages[index - 1].speaker === "tool") {
-                      return null;
-                    }
-                    let end = index;
-                    while (end < messages.length && messages[end].speaker === "tool") {
-                      end++;
-                    }
-                    const rows = messages.slice(index, end) as ToolMessage[];
-                    return (
-                      <ActivityTimeline
-                        key={`tools-${rows[0].run_id}`}
-                        rows={rows}
-                        // The trailing block of an in-flight turn stays
-                        // "working" until the answer starts streaming.
-                        active={isLoading && end === messages.length}
-                        onOpenArtifact={openArtifact}
-                      />
-                    );
-                  }
+				{messages.length === 0 && !messagesLoading ? (
+					<div className="flex-1 flex flex-col items-center justify-center px-4 gap-6">
+						<div className="relative">
+							<div className="absolute -inset-6 bg-amber-soft/20 rounded-full blur-3xl" />
+							<img src={catIcon} alt="Simba" className="relative w-36 h-36" />
+						</div>
+						<h1
+							className="text-2xl font-bold text-charcoal"
+							style={{ fontFamily: "var(--font-display)" }}
+						>
+							Ask me anything
+						</h1>
+						<div className="w-full max-w-2xl">
+							<MessageInput
+								query={query}
+								handleQueryChange={handleQueryChange}
+								handleKeyDown={handleKeyDown}
+								handleQuestionSubmit={handleQuestionSubmit}
+								simbaMode={simbaMode}
+								onToggleSimbaMode={handleToggleSimbaMode}
+								isLoading={isLoading}
+								onStop={stopGeneration}
+								pendingAttachment={pendingAttachment}
+								onAttach={attachFile}
+								onClearAttachment={clearAttachment}
+								attachmentError={attachmentError}
+								onDismissAttachmentError={clearAttachmentError}
+								autoFocus
+							/>
+						</div>
+					</div>
+				) : (
+					<>
+						<div className="flex-1 overflow-y-auto px-4 py-6">
+							<div className="max-w-3xl mx-auto flex flex-col gap-6">
+								{messages.map((msg, index) => {
+									if (msg.speaker === "tool") {
+										// Consecutive tool calls render as one timeline block,
+										// emitted at the first row of the run.
+										if (index > 0 && messages[index - 1].speaker === "tool") {
+											return null;
+										}
+										let end = index;
+										while (
+											end < messages.length &&
+											messages[end].speaker === "tool"
+										) {
+											end++;
+										}
+										const rows = messages.slice(index, end) as ToolMessage[];
+										return (
+											<ActivityTimeline
+												key={`tools-${rows[0].run_id}`}
+												rows={rows}
+												// The trailing block of an in-flight turn stays
+												// "working" until the answer starts streaming.
+												active={isLoading && end === messages.length}
+												onOpenArtifact={openArtifact}
+											/>
+										);
+									}
 
-                  // When the model has been thinking and content is now streaming,
-                  // render ThinkingBubble just before the streaming simba message
-                  // so it appears above the answer rather than below it.
-                  const showThinkingHere =
-                    streaming &&
-                    isLoading &&
-                    (thinking || thinkingText || thinkingDurationMs !== null) &&
-                    index === messages.length - 1 &&
-                    msg.speaker === "simba";
+									// When the model has been thinking and content is now streaming,
+									// render ThinkingBubble just before the streaming simba message
+									// so it appears above the answer rather than below it.
+									const showThinkingHere =
+										streaming &&
+										isLoading &&
+										(thinking || thinkingText || thinkingDurationMs !== null) &&
+										index === messages.length - 1 &&
+										msg.speaker === "simba";
 
-                  const bubble =
-                    msg.speaker === "simba" ? (
-                      <AnswerBubble key={index} text={msg.text} />
-                    ) : (
-                      <QuestionBubble key={index} text={msg.text} image_key={msg.image_key} />
-                    );
+									const bubble =
+										msg.speaker === "simba" ? (
+											<AnswerBubble key={index} text={msg.text} />
+										) : (
+											<QuestionBubble
+												key={index}
+												text={msg.text}
+												image_key={msg.image_key}
+											/>
+										);
 
-                  if (showThinkingHere) {
-                    return (
-                      <React.Fragment key={index}>
-                        <ThinkingBubble
-                          text={thinkingText}
-                          durationMs={thinkingDurationMs}
-                          active={thinking}
-                        />
-                        {bubble}
-                      </React.Fragment>
-                    );
-                  }
-                  return bubble;
-                })}
+									if (showThinkingHere) {
+										return (
+											<React.Fragment key={index}>
+												<ThinkingBubble
+													text={thinkingText}
+													durationMs={thinkingDurationMs}
+													active={thinking}
+												/>
+												{bubble}
+											</React.Fragment>
+										);
+									}
+									return bubble;
+								})}
 
-                {/* ThinkingBubble before content arrives (no streaming message yet) */}
-                {isLoading && !streaming && (thinking || thinkingText || thinkingDurationMs !== null) && (
-                  <ThinkingBubble
-                    text={thinkingText}
-                    durationMs={thinkingDurationMs}
-                    active={thinking}
-                  />
-                )}
+								{/* ThinkingBubble before content arrives (no streaming message yet) */}
+								{isLoading &&
+									!streaming &&
+									(thinking || thinkingText || thinkingDurationMs !== null) && (
+										<ThinkingBubble
+											text={thinkingText}
+											durationMs={thinkingDurationMs}
+											active={thinking}
+										/>
+									)}
 
-                {/* Placeholder dots, unless the activity timeline already shows progress */}
-                {((isLoading &&
-                  !streaming &&
-                  !thinking &&
-                  messages[messages.length - 1]?.speaker !== "tool") ||
-                  messagesLoading) && (
-                  <AnswerBubble text="" loading={true} />
-                )}
+								{/* Placeholder dots, unless the activity timeline already shows progress */}
+								{((isLoading &&
+									!streaming &&
+									!thinking &&
+									messages[messages.length - 1]?.speaker !== "tool") ||
+									messagesLoading) && <AnswerBubble text="" loading={true} />}
 
-                {error && (
-                  <div className="flex justify-center message-enter">
-                    <div className="inline-flex items-center gap-2 px-3 py-1.5 rounded-full bg-red-50 border border-red-200 text-xs text-red-600">
-                      <span>{error}</span>
-                      <button
-                        onClick={clearError}
-                        className="text-red-400 hover:text-red-600 transition-colors cursor-pointer"
-                        aria-label="Dismiss error"
-                      >
-                        <X size={13} />
-                      </button>
-                    </div>
-                  </div>
-                )}
+								{reconnecting && (
+									<p role="status" className="text-sm text-muted-foreground">
+										Connection lost. Reconnecting to your running answer…
+									</p>
+								)}
+								{error && (
+									<div className="flex justify-center message-enter">
+										<div className="inline-flex items-center gap-2 px-3 py-1.5 rounded-full bg-red-50 border border-red-200 text-xs text-red-600">
+											<span>{error}</span>
+											<button
+												onClick={clearError}
+												className="text-red-400 hover:text-red-600 transition-colors cursor-pointer"
+												aria-label="Dismiss error"
+											>
+												<X size={13} />
+											</button>
+										</div>
+									</div>
+								)}
 
-                {stalled && (
-                  <div className="flex justify-center message-enter">
-                    <div className="inline-flex items-center gap-2 px-3 py-1.5 rounded-full bg-red-50 border border-red-200 text-xs text-red-600">
-                      <span>Simba didn&apos;t respond in time.</span>
-                      <button
-                        onClick={retryStalled}
-                        className="font-medium underline underline-offset-2 hover:text-red-800 transition-colors cursor-pointer"
-                      >
-                        Retry
-                      </button>
-                    </div>
-                  </div>
-                )}
+								{stalled && (
+									<div className="flex justify-center message-enter">
+										<div className="inline-flex items-center gap-2 px-3 py-1.5 rounded-full bg-red-50 border border-red-200 text-xs text-red-600">
+											<span>Simba didn&apos;t respond in time.</span>
+											<button
+												onClick={retryStalled}
+												className="font-medium underline underline-offset-2 hover:text-red-800 transition-colors cursor-pointer"
+											>
+												Retry
+											</button>
+										</div>
+									</div>
+								)}
 
-                <div ref={messagesEndRef} />
-              </div>
-            </div>
+								<div ref={messagesEndRef} />
+							</div>
+						</div>
 
-            <footer className="border-t border-sand-light/40 bg-cream">
-              <div className="max-w-3xl mx-auto px-4 py-3">
-                <MessageInput
-                  query={query}
-                  handleQueryChange={handleQueryChange}
-                  handleKeyDown={handleKeyDown}
-                  handleQuestionSubmit={handleQuestionSubmit}
-                  simbaMode={simbaMode}
-                  onToggleSimbaMode={handleToggleSimbaMode}
-                  isLoading={isLoading}
-                  onStop={stopGeneration}
-                  pendingAttachment={pendingAttachment}
-                  onAttach={attachFile}
-                  onClearAttachment={clearAttachment}
-                  attachmentError={attachmentError}
-                  onDismissAttachmentError={clearAttachmentError}
-                />
-              </div>
-            </footer>
-          </>
-        )}
-      </div>
+						<footer className="border-t border-sand-light/40 bg-cream">
+							<div className="max-w-3xl mx-auto px-4 py-3">
+								<MessageInput
+									query={query}
+									handleQueryChange={handleQueryChange}
+									handleKeyDown={handleKeyDown}
+									handleQuestionSubmit={handleQuestionSubmit}
+									simbaMode={simbaMode}
+									onToggleSimbaMode={handleToggleSimbaMode}
+									isLoading={isLoading}
+									onStop={stopGeneration}
+									pendingAttachment={pendingAttachment}
+									onAttach={attachFile}
+									onClearAttachment={clearAttachment}
+									attachmentError={attachmentError}
+									onDismissAttachmentError={clearAttachmentError}
+								/>
+							</div>
+						</footer>
+					</>
+				)}
+			</div>
 
-      <ArtifactPanel panel={artifactPanel} conversationId={conversationId} />
-    </div>
-  );
+			<ArtifactPanel panel={artifactPanel} conversationId={conversationId} />
+		</div>
+	);
 };

raggr-frontend/src/components/ConversationList.tsx

@@ -1,231 +1,322 @@
 import { useEffect, useRef, useState } from "react";
-import { Check, MoreHorizontal, Pencil, Plus, Trash2, X } from "lucide-react";
+import {
+	Download,
+	Pin,
+	PinOff,
+	Check,
+	MoreHorizontal,
+	Pencil,
+	Plus,
+	Trash2,
+	X,
+} from "lucide-react";
 import { cn } from "../lib/utils";
 
-type Conversation = {
-  title: string;
-  id: string;
-};
+import type { Conversation } from "../hooks/useConversations";
 
 type ConversationProps = {
-  conversations: Conversation[];
-  onSelectConversation: (conversation: Conversation) => void;
-  onCreateNewConversation: () => void;
-  onRenameConversation?: (id: string, title: string) => void;
-  onDeleteConversation?: (id: string) => void;
-  selectedId?: string;
-  variant?: "dark" | "light";
+	conversations: Conversation[];
+	onSelectConversation: (conversation: Conversation) => void;
+	onCreateNewConversation: () => void;
+	onRenameConversation?: (id: string, title: string) => void;
+	onDeleteConversation?: (id: string) => void;
+	onPinConversation?: (id: string, pinned: boolean) => void | Promise<void>;
+	onExportConversation?: (
+		id: string,
+		format: "json" | "markdown",
+	) => void | Promise<void>;
+	selectedId?: string;
+	variant?: "dark" | "light";
 };
 
 export const ConversationList = ({
-  conversations,
-  onSelectConversation,
-  onCreateNewConversation,
-  onRenameConversation,
-  onDeleteConversation,
-  selectedId,
-  variant = "dark",
+	conversations,
+	onSelectConversation,
+	onCreateNewConversation,
+	onRenameConversation,
+	onDeleteConversation,
+	onPinConversation,
+	onExportConversation,
+	selectedId,
+	variant = "dark",
 }: ConversationProps) => {
-  const [menuOpenId, setMenuOpenId] = useState<string | null>(null);
-  const [editingId, setEditingId] = useState<string | null>(null);
-  const [editValue, setEditValue] = useState("");
-  const [confirmDeleteId, setConfirmDeleteId] = useState<string | null>(null);
-  const inputRef = useRef<HTMLInputElement>(null);
+	const [menuOpenId, setMenuOpenId] = useState<string | null>(null);
+	const [editingId, setEditingId] = useState<string | null>(null);
+	const [editValue, setEditValue] = useState("");
+	const [confirmDeleteId, setConfirmDeleteId] = useState<string | null>(null);
+	const [actionError, setActionError] = useState("");
+	const runAction = async (action: () => void | Promise<void>) => {
+		setMenuOpenId(null);
+		setActionError("");
+		try {
+			await action();
+		} catch {
+			setActionError("Unable to complete action. Please try again.");
+		}
+	};
+	const inputRef = useRef<HTMLInputElement>(null);
 
-  const isDark = variant === "dark";
+	const isDark = variant === "dark";
 
-  // Close any open row menu when clicking elsewhere.
-  useEffect(() => {
-    if (!menuOpenId && !confirmDeleteId) return;
-    const close = () => {
-      setMenuOpenId(null);
-      setConfirmDeleteId(null);
-    };
-    window.addEventListener("click", close);
-    return () => window.removeEventListener("click", close);
-  }, [menuOpenId, confirmDeleteId]);
+	// Close any open row menu when clicking elsewhere.
+	useEffect(() => {
+		if (!menuOpenId && !confirmDeleteId) return;
+		const close = () => {
+			setMenuOpenId(null);
+			setConfirmDeleteId(null);
+		};
+		const escape = (event: KeyboardEvent) => {
+			if (event.key === "Escape") close();
+		};
+		window.addEventListener("click", close);
+		window.addEventListener("keydown", escape);
+		return () => {
+			window.removeEventListener("click", close);
+			window.removeEventListener("keydown", escape);
+		};
+	}, [menuOpenId, confirmDeleteId]);
 
-  useEffect(() => {
-    if (editingId) inputRef.current?.focus();
-  }, [editingId]);
+	useEffect(() => {
+		if (editingId) inputRef.current?.focus();
+	}, [editingId]);
 
-  const startRename = (conv: Conversation) => {
-    setMenuOpenId(null);
-    setEditingId(conv.id);
-    setEditValue(conv.title);
-  };
+	const startRename = (conv: Conversation) => {
+		setMenuOpenId(null);
+		setEditingId(conv.id);
+		setEditValue(conv.title);
+	};
 
-  const commitRename = () => {
-    if (editingId && editValue.trim()) {
-      onRenameConversation?.(editingId, editValue.trim());
-    }
-    setEditingId(null);
-  };
+	const commitRename = () => {
+		if (editingId && editValue.trim()) {
+			onRenameConversation?.(editingId, editValue.trim());
+		}
+		setEditingId(null);
+	};
 
-  const hasActions = Boolean(onRenameConversation || onDeleteConversation);
+	const hasActions = Boolean(
+		onRenameConversation ||
+			onDeleteConversation ||
+			onPinConversation ||
+			onExportConversation,
+	);
 
-  return (
-    <div className="flex flex-col gap-1">
-      <button
-        onClick={onCreateNewConversation}
-        className={cn(
-          "flex items-center gap-2 w-full px-3 py-2 rounded-xl",
-          "text-sm transition-all duration-150 cursor-pointer mb-1",
-          isDark
-            ? "text-cream/60 hover:text-cream hover:bg-white/8"
-            : "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
-        )}
-      >
-        <Plus size={14} strokeWidth={2.5} />
-        <span>New thread</span>
-      </button>
+	return (
+		<div className="flex flex-col gap-1">
+			{actionError && (
+				<p role="alert" className="text-sm text-red-400">
+					{actionError}
+				</p>
+			)}
+			<button
+				onClick={onCreateNewConversation}
+				className={cn(
+					"flex items-center gap-2 w-full px-3 py-2 rounded-xl",
+					"text-sm transition-all duration-150 cursor-pointer mb-1",
+					isDark
+						? "text-cream/60 hover:text-cream hover:bg-white/8"
+						: "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
+				)}
+			>
+				<Plus size={14} strokeWidth={2.5} />
+				<span>New thread</span>
+			</button>
 
-      {conversations.map((conv) => {
-        const isActive = conv.id === selectedId;
-        const isEditing = editingId === conv.id;
+			{conversations.map((conv) => {
+				const isActive = conv.id === selectedId;
+				const isEditing = editingId === conv.id;
 
-        if (isEditing) {
-          return (
-            <div
-              key={conv.id}
-              className={cn(
-                "flex items-center gap-1 px-1.5 py-1 rounded-xl",
-                isDark ? "bg-white/12" : "bg-cream-dark",
-              )}
-            >
-              <input
-                ref={inputRef}
-                value={editValue}
-                onChange={(e) => setEditValue(e.target.value)}
-                onKeyDown={(e) => {
-                  if (e.key === "Enter") commitRename();
-                  if (e.key === "Escape") setEditingId(null);
-                }}
-                className={cn(
-                  "flex-1 min-w-0 bg-transparent px-1.5 py-1 text-sm rounded-lg",
-                  "focus:outline-none",
-                  isDark
-                    ? "text-cream placeholder:text-cream/40"
-                    : "text-charcoal placeholder:text-warm-gray/50",
-                )}
-              />
-              <button
-                onClick={commitRename}
-                className={cn(
-                  "w-6 h-6 rounded-lg flex items-center justify-center shrink-0 cursor-pointer transition-colors",
-                  isDark
-                    ? "text-cream/60 hover:text-cream hover:bg-white/10"
-                    : "text-warm-gray hover:text-charcoal hover:bg-sand-light",
-                )}
-              >
-                <Check size={14} />
-              </button>
-              <button
-                onClick={() => setEditingId(null)}
-                className={cn(
-                  "w-6 h-6 rounded-lg flex items-center justify-center shrink-0 cursor-pointer transition-colors",
-                  isDark
-                    ? "text-cream/60 hover:text-cream hover:bg-white/10"
-                    : "text-warm-gray hover:text-charcoal hover:bg-sand-light",
-                )}
-              >
-                <X size={14} />
-              </button>
-            </div>
-          );
-        }
+				if (isEditing) {
+					return (
+						<div
+							key={conv.id}
+							className={cn(
+								"flex items-center gap-1 px-1.5 py-1 rounded-xl",
+								isDark ? "bg-white/12" : "bg-cream-dark",
+							)}
+						>
+							<input
+								ref={inputRef}
+								value={editValue}
+								onChange={(e) => setEditValue(e.target.value)}
+								onKeyDown={(e) => {
+									if (e.key === "Enter") commitRename();
+									if (e.key === "Escape") setEditingId(null);
+								}}
+								className={cn(
+									"flex-1 min-w-0 bg-transparent px-1.5 py-1 text-sm rounded-lg",
+									"focus:outline-none",
+									isDark
+										? "text-cream placeholder:text-cream/40"
+										: "text-charcoal placeholder:text-warm-gray/50",
+								)}
+							/>
+							<button
+								onClick={commitRename}
+								className={cn(
+									"w-6 h-6 rounded-lg flex items-center justify-center shrink-0 cursor-pointer transition-colors",
+									isDark
+										? "text-cream/60 hover:text-cream hover:bg-white/10"
+										: "text-warm-gray hover:text-charcoal hover:bg-sand-light",
+								)}
+							>
+								<Check size={14} />
+							</button>
+							<button
+								onClick={() => setEditingId(null)}
+								className={cn(
+									"w-6 h-6 rounded-lg flex items-center justify-center shrink-0 cursor-pointer transition-colors",
+									isDark
+										? "text-cream/60 hover:text-cream hover:bg-white/10"
+										: "text-warm-gray hover:text-charcoal hover:bg-sand-light",
+								)}
+							>
+								<X size={14} />
+							</button>
+						</div>
+					);
+				}
 
-        return (
-          <div key={conv.id} className="group relative flex items-center">
-            <button
-              onClick={() => onSelectConversation(conv)}
-              className={cn(
-                "flex-1 min-w-0 px-3 py-2 rounded-xl text-left",
-                "text-sm truncate transition-all duration-150 cursor-pointer",
-                hasActions && "pr-8",
-                isDark
-                  ? isActive
-                    ? "bg-white/12 text-cream font-medium"
-                    : "text-cream/60 hover:text-cream hover:bg-white/8"
-                  : isActive
-                    ? "bg-cream-dark text-charcoal font-medium"
-                    : "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
-              )}
-            >
-              {conv.title}
-            </button>
+				return (
+					<div key={conv.id} className="group relative flex items-center">
+						<button
+							onClick={() => onSelectConversation(conv)}
+							className={cn(
+								"flex-1 min-w-0 px-3 py-2 rounded-xl text-left",
+								"text-sm truncate transition-all duration-150 cursor-pointer",
+								hasActions && "pr-8",
+								isDark
+									? isActive
+										? "bg-white/12 text-cream font-medium"
+										: "text-cream/60 hover:text-cream hover:bg-white/8"
+									: isActive
+										? "bg-cream-dark text-charcoal font-medium"
+										: "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
+							)}
+						>
+							<span className="block truncate">
+								{conv.pinned && (
+									<Pin size={12} className="inline mr-1" aria-label="Pinned" />
+								)}
+								{conv.title}
+							</span>
+							{conv.snippet && (
+								<span className="block text-xs opacity-75 whitespace-normal line-clamp-2">
+									{conv.snippet}
+								</span>
+							)}
+						</button>
 
-            {hasActions && (
-              <button
-                onClick={(e) => {
-                  e.stopPropagation();
-                  setConfirmDeleteId(null);
-                  setMenuOpenId((prev) => (prev === conv.id ? null : conv.id));
-                }}
-                className={cn(
-                  "absolute right-1.5 w-6 h-6 rounded-lg flex items-center justify-center cursor-pointer transition-all",
-                  "opacity-0 group-hover:opacity-100 focus:opacity-100",
-                  menuOpenId === conv.id && "opacity-100",
-                  isDark
-                    ? "text-cream/50 hover:text-cream hover:bg-white/10"
-                    : "text-warm-gray hover:text-charcoal hover:bg-sand-light",
-                )}
-              >
-                <MoreHorizontal size={15} />
-              </button>
-            )}
+						{hasActions && (
+							<button
+								aria-label={`Actions for ${conv.title}`}
+								aria-expanded={menuOpenId === conv.id}
+								onClick={(e) => {
+									e.stopPropagation();
+									setConfirmDeleteId(null);
+									setMenuOpenId((prev) => (prev === conv.id ? null : conv.id));
+								}}
+								className={cn(
+									"absolute right-1.5 w-6 h-6 rounded-lg flex items-center justify-center cursor-pointer transition-all",
+									"opacity-0 group-hover:opacity-100 focus:opacity-100",
+									menuOpenId === conv.id && "opacity-100",
+									isDark
+										? "text-cream/50 hover:text-cream hover:bg-white/10"
+										: "text-warm-gray hover:text-charcoal hover:bg-sand-light",
+								)}
+							>
+								<MoreHorizontal size={15} />
+							</button>
+						)}
 
-            {menuOpenId === conv.id && (
-              <div
-                onClick={(e) => e.stopPropagation()}
-                className={cn(
-                  "absolute right-1.5 top-9 z-20 w-36 rounded-xl p-1 shadow-lg",
-                  isDark
-                    ? "bg-sidebar-bg border border-white/10"
-                    : "bg-warm-white border border-sand-light",
-                )}
-              >
-                {onRenameConversation && (
-                  <button
-                    onClick={() => startRename(conv)}
-                    className={cn(
-                      "flex items-center gap-2 w-full px-2.5 py-1.5 rounded-lg text-sm cursor-pointer transition-colors",
-                      isDark
-                        ? "text-cream/70 hover:text-cream hover:bg-white/10"
-                        : "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
-                    )}
-                  >
-                    <Pencil size={13} />
-                    <span>Rename</span>
-                  </button>
-                )}
-                {onDeleteConversation && (
-                  <button
-                    onClick={() => {
-                      if (confirmDeleteId === conv.id) {
-                        onDeleteConversation(conv.id);
-                        setMenuOpenId(null);
-                        setConfirmDeleteId(null);
-                      } else {
-                        setConfirmDeleteId(conv.id);
-                      }
-                    }}
-                    className={cn(
-                      "flex items-center gap-2 w-full px-2.5 py-1.5 rounded-lg text-sm cursor-pointer transition-colors",
-                      "text-red-400 hover:bg-red-500/10",
-                    )}
-                  >
-                    <Trash2 size={13} />
-                    <span>
-                      {confirmDeleteId === conv.id ? "Confirm delete" : "Delete"}
-                    </span>
-                  </button>
-                )}
-              </div>
-            )}
-          </div>
-        );
-      })}
-    </div>
-  );
+						{menuOpenId === conv.id && (
+							<div
+								onClick={(e) => e.stopPropagation()}
+								className={cn(
+									"absolute right-1.5 top-9 z-20 w-36 rounded-xl p-1 shadow-lg",
+									isDark
+										? "bg-sidebar-bg border border-white/10"
+										: "bg-warm-white border border-sand-light",
+								)}
+							>
+								{onPinConversation && (
+									<button
+										className={cn(
+											"flex items-center gap-2 w-full px-2.5 py-1.5 rounded-lg text-sm cursor-pointer",
+											isDark
+												? "text-cream/70 hover:text-cream hover:bg-white/10"
+												: "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
+										)}
+										aria-pressed={!!conv.pinned}
+										aria-label={`${conv.pinned ? "Unpin" : "Pin"} ${conv.title}`}
+										onClick={() =>
+											runAction(() => onPinConversation(conv.id, !conv.pinned))
+										}
+									>
+										{conv.pinned ? <PinOff size={13} /> : <Pin size={13} />}
+										{conv.pinned ? "Unpin" : "Pin"}
+									</button>
+								)}
+								{onExportConversation &&
+									(["json", "markdown"] as const).map((format) => (
+										<button
+											key={format}
+											className={cn(
+												"flex items-center gap-2 w-full px-2.5 py-1.5 rounded-lg text-sm cursor-pointer",
+												isDark
+													? "text-cream/70 hover:text-cream hover:bg-white/10"
+													: "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
+											)}
+											onClick={() =>
+												runAction(() => onExportConversation(conv.id, format))
+											}
+										>
+											<Download size={13} /> Export{" "}
+											{format === "json" ? "JSON" : "Markdown"}
+										</button>
+									))}
+								{onRenameConversation && (
+									<button
+										onClick={() => startRename(conv)}
+										className={cn(
+											"flex items-center gap-2 w-full px-2.5 py-1.5 rounded-lg text-sm cursor-pointer transition-colors",
+											isDark
+												? "text-cream/70 hover:text-cream hover:bg-white/10"
+												: "text-warm-gray hover:text-charcoal hover:bg-cream-dark",
+										)}
+									>
+										<Pencil size={13} />
+										<span>Rename</span>
+									</button>
+								)}
+								{onDeleteConversation && (
+									<button
+										onClick={() => {
+											if (confirmDeleteId === conv.id) {
+												onDeleteConversation(conv.id);
+												setMenuOpenId(null);
+												setConfirmDeleteId(null);
+											} else {
+												setConfirmDeleteId(conv.id);
+											}
+										}}
+										className={cn(
+											"flex items-center gap-2 w-full px-2.5 py-1.5 rounded-lg text-sm cursor-pointer transition-colors",
+											"text-red-400 hover:bg-red-500/10",
+										)}
+									>
+										<Trash2 size={13} />
+										<span>
+											{confirmDeleteId === conv.id
+												? "Confirm delete"
+												: "Delete"}
+										</span>
+									</button>
+								)}
+							</div>
+						)}
+					</div>
+				);
+			})}
+		</div>
+	);
 };

raggr-frontend/src/components/ConversationsPage.tsx

@@ -8,87 +8,109 @@ import { Input } from "./ui/input";
 import catIcon from "../assets/cat.png";
 
 export const ConversationsPage = () => {
-  const navigate = useNavigate();
-  const {
-    items,
-    search,
-    setSearch,
-    hasMore,
-    loading,
-    loadMore,
-    renameConversation,
-    deleteConversation,
-  } = useConversationBrowser();
+	const navigate = useNavigate();
+	const {
+		items,
+		error,
+		pinConversation,
+		exportConversation,
+		search,
+		setSearch,
+		hasMore,
+		loading,
+		loadMore,
+		renameConversation,
+		deleteConversation,
+	} = useConversationBrowser();
 
-  const handleSelect = useCallback(
-    (conversation: Conversation) => navigate(`/c/${conversation.id}`),
-    [navigate],
-  );
+	const handleSelect = useCallback(
+		(conversation: Conversation) =>
+			navigate(
+				`/c/${conversation.id}${conversation.matchMessageId ? `?message=${encodeURIComponent(conversation.matchMessageId)}` : ""}`,
+			),
+		[navigate],
+	);
 
-  const handleCreateNew = useCallback(() => navigate("/"), [navigate]);
+	const handleCreateNew = useCallback(() => navigate("/"), [navigate]);
 
-  return (
-    <div className="h-screen h-[100dvh] flex flex-col bg-cream overflow-hidden">
-      <header className="flex items-center gap-3 px-4 py-3 bg-warm-white border-b border-sand-light/60">
-        <button
-          onClick={() => navigate(-1)}
-          className="w-8 h-8 rounded-xl flex items-center justify-center text-warm-gray hover:text-charcoal hover:bg-cream-dark transition-all cursor-pointer"
-          aria-label="Back"
-        >
-          <ArrowLeft size={16} />
-        </button>
-        <img src={catIcon} alt="Simba" className="w-10 h-10" />
-        <h1
-          className="text-base font-bold text-charcoal"
-          style={{ fontFamily: "var(--font-display)" }}
-        >
-          All conversations
-        </h1>
-      </header>
+	return (
+		<div className="h-screen h-[100dvh] flex flex-col bg-cream overflow-hidden">
+			<header className="flex items-center gap-3 px-4 py-3 bg-warm-white border-b border-sand-light/60">
+				<button
+					onClick={() => navigate(-1)}
+					className="w-8 h-8 rounded-xl flex items-center justify-center text-warm-gray hover:text-charcoal hover:bg-cream-dark transition-all cursor-pointer"
+					aria-label="Back"
+				>
+					<ArrowLeft size={16} />
+				</button>
+				<img src={catIcon} alt="Simba" className="w-10 h-10" />
+				<h1
+					className="text-base font-bold text-charcoal"
+					style={{ fontFamily: "var(--font-display)" }}
+				>
+					All conversations
+				</h1>
+			</header>
 
-      <div className="flex-1 overflow-y-auto">
-        <div className="max-w-2xl mx-auto px-4 py-6 flex flex-col gap-4">
-          <div className="relative">
-            <Search
-              size={15}
-              className="absolute left-3 top-1/2 -translate-y-1/2 text-warm-gray/60"
-            />
-            <Input
-              value={search}
-              onChange={(e) => setSearch(e.target.value)}
-              placeholder="Search conversations..."
-              className="h-10 pl-9"
-            />
-          </div>
+			<div className="flex-1 overflow-y-auto">
+				<div className="max-w-2xl mx-auto px-4 py-6 flex flex-col gap-4">
+					<div className="relative">
+						<Search
+							size={15}
+							className="absolute left-3 top-1/2 -translate-y-1/2 text-warm-gray/60"
+						/>
+						<Input
+							value={search}
+							onChange={(e) => setSearch(e.target.value)}
+							placeholder="Search titles and messages..."
+							aria-label="Search conversation titles and messages"
+							maxLength={500}
+							className="h-10 pl-9"
+						/>
+					</div>
 
-          <div className="bg-warm-white rounded-2xl border border-sand-light p-3 shadow-sm">
-            <ConversationList
-              conversations={items}
-              onCreateNewConversation={handleCreateNew}
-              onSelectConversation={handleSelect}
-              onRenameConversation={renameConversation}
-              onDeleteConversation={deleteConversation}
-              variant="light"
-            />
+					{error && (
+						<p role="alert" className="text-sm text-red-600">
+							{error}
+						</p>
+					)}
+					{loading && (
+						<p role="status" className="text-sm text-warm-gray">
+							Loading conversations...
+						</p>
+					)}
+					<div className="bg-warm-white rounded-2xl border border-sand-light p-3 shadow-sm">
+						<ConversationList
+							conversations={items}
+							onCreateNewConversation={handleCreateNew}
+							onSelectConversation={handleSelect}
+							onPinConversation={pinConversation}
+							onExportConversation={exportConversation}
+							onRenameConversation={renameConversation}
+							onDeleteConversation={deleteConversation}
+							variant="light"
+						/>
 
-            {items.length === 0 && !loading && (
-              <p className="text-center text-sm text-warm-gray/70 py-6">
-                {search ? "No conversations match your search." : "No conversations yet."}
-              </p>
-            )}
-          </div>
+						{items.length === 0 && !loading && (
+							<p className="text-center text-sm text-warm-gray/70 py-6">
+								{search
+									? "No conversations match your search."
+									: "No conversations yet."}
+							</p>
+						)}
+					</div>
 
-          {hasMore && (
-            <button
-              onClick={loadMore}
-              disabled={loading}
-              className="self-center px-4 py-2 rounded-xl text-sm text-warm-gray hover:text-charcoal hover:bg-cream-dark transition-all cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed"
-            >
-              {loading ? "Loading..." : "Load more"}
-            </button>
-          )}
-        </div>
-      </div>
-    </div>
-  );
+					{hasMore && (
+						<button
+							onClick={loadMore}
+							disabled={loading}
+							className="self-center px-4 py-2 rounded-xl text-sm text-warm-gray hover:text-charcoal hover:bg-cream-dark transition-all cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed"
+						>
+							{loading ? "Loading..." : "Load more"}
+						</button>
+					)}
+				</div>
+			</div>
+		</div>
+	);
 };

raggr-frontend/src/components/IntegrationStatusPanel.tsx

@@ -0,0 +1,57 @@
+import { useEffect, useState } from "react";
+import { X, RefreshCw } from "lucide-react";
+import { userService } from "../api/userService";
+
+type Status = { name: string; status: string; message: string; last_success_at: string | null; indexed_chunks?: number };
+
+export const IntegrationStatusPanel = ({ onClose }: { onClose: () => void }) => {
+  const [items, setItems] = useState<Status[]>([]);
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState("");
+  const [revision, setRevision] = useState(0);
+  useEffect(() => {
+    const controller = new AbortController();
+    setLoading(true);
+    setError("");
+    void (async () => {
+      try {
+        const response = await userService.fetchWithAuth("/api/integrations/", { signal: controller.signal });
+        if (!response.ok) throw new Error("Unable to load integration status.");
+        const body = await response.json();
+        if (!controller.signal.aborted) setItems(body.integrations);
+      } catch (err) {
+        if (!controller.signal.aborted) setError(err instanceof Error ? err.message : "Unable to load integration status.");
+      } finally {
+        if (!controller.signal.aborted) setLoading(false);
+      }
+    })();
+    return () => controller.abort();
+  }, [revision]);
+  useEffect(() => {
+    const close = (event: KeyboardEvent) => { if (event.key === "Escape") onClose(); };
+    window.addEventListener("keydown", close);
+    return () => window.removeEventListener("keydown", close);
+  }, [onClose]);
+  return (
+    <div className="fixed inset-0 z-50 bg-charcoal/30 flex items-center justify-center p-4" onClick={onClose}>
+      <section role="dialog" aria-modal="true" aria-labelledby="integration-status-title" className="bg-cream rounded-2xl shadow-xl w-full max-w-xl max-h-[85vh] overflow-y-auto p-5" onClick={(e) => e.stopPropagation()}>
+        <header className="flex items-center justify-between gap-3 mb-4">
+          <h2 id="integration-status-title" className="font-semibold text-charcoal">Integrations and data freshness</h2>
+          <div className="flex gap-2">
+            <button aria-label="Refresh integration status" disabled={loading} onClick={() => setRevision((r) => r + 1)} className="p-2 rounded-lg hover:bg-cream-dark"><RefreshCw size={18} /></button>
+            <button aria-label="Close integration status" onClick={onClose} className="p-2 rounded-lg hover:bg-cream-dark"><X size={18} /></button>
+          </div>
+        </header>
+        {loading && <p role="status" className="text-sm text-warm-gray">Checking data freshness…</p>}
+        {error && <p role="alert" className="text-sm text-red-700">{error}</p>}
+        <ul className="space-y-3">
+          {items.map((item) => <li key={item.name} className="border border-sand-light rounded-xl p-3">
+            <div className="flex justify-between gap-2"><span className="font-medium">{item.name}</span><span className={item.status === "ready" ? "text-leaf-dark text-sm" : item.status === "stale" || item.status === "unavailable" ? "text-amber-800 text-sm" : "text-warm-gray text-sm"}>{item.status.replace(/_/g, " ")}</span></div>
+            <p className="text-sm text-warm-gray mt-1">{item.message}</p>
+            {item.last_success_at && <p className="text-xs text-warm-gray mt-1">Last synchronized: {new Date(item.last_success_at).toLocaleString()}</p>}
+          </li>)}
+        </ul>
+      </section>
+    </div>
+  );
+};

raggr-frontend/src/components/MemoryPanel.tsx

@@ -0,0 +1,222 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+import { historyService, type SavedMemory } from "../api/historyService";
+
+type MemoryPanelProps = { onClose?: () => void };
+
+/** Mount inside a page or dialog. Only the authenticated user's saved memories are loaded. */
+export function MemoryPanel({ onClose }: MemoryPanelProps) {
+	const [memories, setMemories] = useState<SavedMemory[]>([]);
+	const [loading, setLoading] = useState(true);
+	const [hasMore, setHasMore] = useState(false);
+	const [error, setError] = useState("");
+	const [busy, setBusy] = useState<string | null>(null);
+	const [editing, setEditing] = useState<string | null>(null);
+	const [content, setContent] = useState("");
+	const [forgetting, setForgetting] = useState<string | null>(null);
+	const [notice, setNotice] = useState("");
+	const offset = useRef(0);
+	const generation = useRef(0);
+
+	const load = useCallback(async (append = false) => {
+		const requestId = ++generation.current;
+		setLoading(true);
+		setError("");
+		try {
+			const page = await historyService.memories(append ? offset.current : 0);
+			if (requestId !== generation.current) return;
+			offset.current = (append ? offset.current : 0) + page.length;
+			setMemories((previous) =>
+				append
+					? [
+							...previous,
+							...page.filter((m) => !previous.some((old) => old.id === m.id)),
+						]
+					: page,
+			);
+			setHasMore(page.length === 100);
+		} catch {
+			if (requestId === generation.current)
+				setError("Unable to load memories. Please try again.");
+		} finally {
+			if (requestId === generation.current) setLoading(false);
+		}
+	}, []);
+	useEffect(() => {
+		void load();
+		return () => {
+			++generation.current;
+		};
+	}, [load]);
+
+	const save = async (memory: SavedMemory) => {
+		setBusy(memory.id);
+		setError("");
+		try {
+			await historyService.correctMemory(memory.id, content.trim());
+			setEditing(null);
+			setNotice(
+				"Memory corrected. Future responses will use the saved correction.",
+			);
+			await load();
+		} catch {
+			setError(
+				"Unable to correct memory. Your edit is still here; please try again.",
+			);
+		} finally {
+			setBusy(null);
+		}
+	};
+	const forget = async (memory: SavedMemory) => {
+		setBusy(memory.id);
+		setError("");
+		try {
+			await historyService.forgetMemory(memory.id);
+			setForgetting(null);
+			if (editing === memory.id) setEditing(null);
+			setNotice(
+				"Memory forgotten. Existing conversation messages are retained.",
+			);
+			await load();
+		} catch {
+			setError("Unable to forget memory. Please try again.");
+		} finally {
+			setBusy(null);
+		}
+	};
+
+	return (
+		<section
+			aria-label="Saved memories"
+			className="bg-warm-white text-charcoal rounded-2xl p-4 flex flex-col gap-3 max-h-[80dvh] overflow-y-auto"
+		>
+			<div className="flex items-center justify-between gap-4">
+				<h2 className="font-bold text-lg">Saved memories</h2>
+				{onClose && (
+					<button
+						type="button"
+						onClick={onClose}
+						className="px-3 py-2 rounded-xl hover:bg-cream-dark"
+					>
+						Close
+					</button>
+				)}
+			</div>
+			<p className="text-sm text-warm-gray">
+				Correct facts Simba remembers about you, or forget a saved memory.
+			</p>
+			{error && (
+				<div role="alert">
+					<p>{error}</p>
+					<button
+						type="button"
+						disabled={loading || !!busy}
+						onClick={() => load()}
+					>
+						Retry loading
+					</button>
+				</div>
+			)}
+			{notice && (
+				<p role="status" className="text-sm">
+					{notice}
+				</p>
+			)}
+			{loading && <p role="status">Loading memories...</p>}
+			{!loading && !error && memories.length === 0 && (
+				<p>No saved memories yet.</p>
+			)}
+			{memories.map((memory) => (
+				<div
+					key={memory.id}
+					className="border border-sand-light rounded-xl p-3 flex flex-col gap-2"
+				>
+					{editing === memory.id ? (
+						<>
+							<label htmlFor={`memory-${memory.id}`} className="text-sm">
+								Correct memory
+							</label>
+							<textarea
+								id={`memory-${memory.id}`}
+								autoFocus
+								maxLength={10000}
+								value={content}
+								onChange={(e) => setContent(e.target.value)}
+								className="border border-sand-light rounded-lg p-2 min-h-24"
+							/>
+							<div className="flex gap-3">
+								<button
+									type="button"
+									disabled={!!busy || !content.trim()}
+									onClick={() => save(memory)}
+								>
+									Save correction
+								</button>
+								<button
+									type="button"
+									disabled={!!busy}
+									onClick={() => setEditing(null)}
+								>
+									Cancel
+								</button>
+							</div>
+						</>
+					) : (
+						<p className="whitespace-pre-wrap break-words">{memory.content}</p>
+					)}
+					<div className="flex gap-3 text-sm">
+						{editing !== memory.id && (
+							<button
+								type="button"
+								disabled={loading || !!busy}
+								onClick={() => {
+									setContent(memory.content);
+									setEditing(memory.id);
+									setForgetting(null);
+								}}
+							>
+								Correct
+							</button>
+						)}
+						<button
+							type="button"
+							disabled={loading || !!busy}
+							onClick={() => setForgetting(memory.id)}
+						>
+							Forget
+						</button>
+					</div>
+					{forgetting === memory.id && (
+						<div className="text-sm flex flex-col gap-2">
+							<p>Forget this saved memory?</p>
+							<div className="flex gap-3">
+								<button
+									type="button"
+									disabled={!!busy}
+									onClick={() => forget(memory)}
+								>
+									Confirm forget
+								</button>
+								<button
+									type="button"
+									disabled={!!busy}
+									onClick={() => setForgetting(null)}
+								>
+									Cancel
+								</button>
+							</div>
+						</div>
+					)}
+				</div>
+			))}
+			{hasMore && (
+				<button
+					type="button"
+					disabled={loading || !!busy}
+					onClick={() => load(true)}
+				>
+					Load more memories
+				</button>
+			)}
+		</section>
+	);
+}

raggr-frontend/src/components/ScheduledMessagesPanel.tsx

@@ -1,4 +1,4 @@
-import { useState } from "react";
+import { useEffect, useRef, useState } from "react";
 import { X, Clock, Send, Trash2, XCircle, RotateCcw, Repeat, Bot } from "lucide-react";
 import { cn } from "../lib/utils";
 import { Button } from "./ui/button";
@@ -15,6 +15,10 @@ import {
 import { useScheduledMessages } from "../hooks/useScheduledMessages";
 import {
   scheduledMessageService,
+  canRetryScheduledMessage,
+  type DeliveryStatus,
+  type ScheduledDelivery,
+  type ScheduledMessage,
   type CreateScheduledMessage,
 } from "../api/scheduledMessageService";
 
@@ -24,13 +28,56 @@ type Props = {
 
 const STATUS_BADGE: Record<string, "amber" | "default" | "destructive" | "muted"> = {
   pending: "amber",
+  processing: "amber",
+  sending: "amber",
+  uncertain: "destructive",
   sent: "default",
   failed: "destructive",
   cancelled: "muted",
 };
 
+export const DELIVERY_STATUS_LABELS: Record<DeliveryStatus, string> = {
+  pending: "Pending",
+  processing: "Processing",
+  sending: "Sending",
+  sent: "Sent",
+  failed: "Failed",
+  uncertain: "Uncertain — needs reconciliation",
+  cancelled: "Cancelled",
+};
+
+export const UNCERTAIN_DELIVERY_HELP = "Delivery may already have occurred. Reconcile with the provider or recipient before any resend; uncertain deliveries cannot be retried.";
+
+export function DeliveryHistory({ deliveries }: { deliveries: ScheduledDelivery[] }) {
+  if (deliveries.length === 0) return <p className="text-xs text-warm-gray">No delivery attempts yet.</p>;
+  return (
+    <div className="space-y-3">
+      {deliveries.map((delivery) => (
+        <div key={delivery.id} className="rounded-xl border border-sand-light p-3 text-xs space-y-2">
+          <p className="break-all">Run: {delivery.run_id}</p>
+          <Badge variant={STATUS_BADGE[delivery.status]}>{DELIVERY_STATUS_LABELS[delivery.status]}</Badge>
+          {delivery.error && <p className="text-red-500 break-words">{delivery.error}</p>}
+          {delivery.status === "uncertain" && <p>{UNCERTAIN_DELIVERY_HELP}</p>}
+          {delivery.attempts.length === 0 ? <p>No attempts recorded.</p> : (
+            <ol className="space-y-2">
+              {delivery.attempts.map((attempt, index) => (
+                <li key={attempt.id}>
+                  <p>Attempt {index + 1}: {DELIVERY_STATUS_LABELS[attempt.status]}</p>
+                  <p>Started: {new Date(attempt.started_at).toLocaleString()}</p>
+                  <p>{attempt.finished_at ? `Finished: ${new Date(attempt.finished_at).toLocaleString()}` : "Not finished"}</p>
+                  {attempt.error && <p className="text-red-500 break-words">{attempt.error}</p>}
+                </li>
+              ))}
+            </ol>
+          )}
+        </div>
+      ))}
+    </div>
+  );
+}
+
 export const ScheduledMessagesPanel = ({ onClose }: Props) => {
-  const { messages, loading, refresh } = useScheduledMessages();
+  const { messages, loading, error: listError, refresh } = useScheduledMessages();
   const [channel, setChannel] = useState<"imessage" | "email">("imessage");
   const [recipient, setRecipient] = useState("");
   const [subject, setSubject] = useState("");
@@ -40,6 +87,28 @@ export const ScheduledMessagesPanel = ({ onClose }: Props) => {
   const [useAgent, setUseAgent] = useState(false);
   const [error, setError] = useState("");
   const [submitting, setSubmitting] = useState(false);
+  const [busyId, setBusyId] = useState<string | null>(null);
+  const actionInFlight = useRef(false);
+  const [historyId, setHistoryId] = useState<string | null>(null);
+  const [deliveries, setDeliveries] = useState<ScheduledDelivery[]>([]);
+  const [historyLoading, setHistoryLoading] = useState(false);
+  const [historyError, setHistoryError] = useState("");
+  const [historyVersion, setHistoryVersion] = useState(0);
+
+  useEffect(() => {
+    if (!historyId) return;
+    let active = true;
+    setHistoryLoading(true);
+    setHistoryError("");
+    setDeliveries([]);
+    scheduledMessageService.deliveries(historyId)
+      .then((result) => { if (active) setDeliveries(result); })
+      .catch((err) => {
+        if (active) setHistoryError(err instanceof Error ? err.message : "Failed to load delivery history");
+      })
+      .finally(() => { if (active) setHistoryLoading(false); });
+    return () => { active = false; };
+  }, [historyId, historyVersion]);
 
   const handleCreate = async () => {
     setError("");
@@ -78,26 +147,31 @@ export const ScheduledMessagesPanel = ({ onClose }: Props) => {
     }
   };
 
-  const handleCancel = async (id: string) => {
+  const runAction = async (message: ScheduledMessage, action: "cancel" | "delete" | "retry") => {
+    if (actionInFlight.current) return;
+    if (action === "retry" && !canRetryScheduledMessage(message)) {
+      setError("Only failed, retryable deliveries can be retried.");
+      return;
+    }
+    actionInFlight.current = true;
+    setBusyId(message.id);
+    setError("");
     try {
-      await scheduledMessageService.update(id, { status: "cancelled" });
-      refresh();
-    } catch {}
-  };
-
-  const handleDelete = async (id: string) => {
-    try {
-      await scheduledMessageService.remove(id);
-      refresh();
-    } catch {}
-  };
-
-  const handleRetry = async (id: string) => {
-    try {
-      const futureTime = new Date(Date.now() + 30_000).toISOString();
-      await scheduledMessageService.update(id, { scheduled_at: futureTime });
-      refresh();
-    } catch {}
+      if (action === "cancel") await scheduledMessageService.update(message.id, { status: "cancelled" });
+      if (action === "delete") {
+        await scheduledMessageService.remove(message.id);
+        if (historyId === message.id) setHistoryId(null);
+      }
+      if (action === "retry") await scheduledMessageService.retry(message.id);
+    } catch (err) {
+      setError(err instanceof Error ? err.message : `Failed to ${action} scheduled message`);
+    } finally {
+      // A rejected retry can still have changed the delivery to failed or uncertain.
+      await refresh();
+      setHistoryVersion((version) => version + 1);
+      actionInFlight.current = false;
+      setBusyId(null);
+    }
   };
 
   return (
@@ -226,7 +300,7 @@ export const ScheduledMessagesPanel = ({ onClose }: Props) => {
               className="w-full rounded-xl border border-sand bg-cream-light px-3 py-2 text-sm text-charcoal placeholder:text-warm-gray/50 focus:outline-none focus:ring-2 focus:ring-leaf/30 resize-none"
             />
 
-            {error && <p className="text-xs text-red-500">{error}</p>}
+            {error && <p role="alert" className="text-xs text-red-500">{error}</p>}
 
             <Button onClick={handleCreate} disabled={submitting} size="sm">
               <Send size={12} />
@@ -235,6 +309,11 @@ export const ScheduledMessagesPanel = ({ onClose }: Props) => {
           </div>
 
           {/* Message list */}
+          {listError && (
+            <div role="alert" className="px-6 py-3 text-xs text-red-500">
+              {listError} <Button size="sm" variant="ghost-dark" onClick={() => void refresh()}>Refresh messages</Button>
+            </div>
+          )}
           {loading ? (
             <div className="px-6 py-12 text-center text-warm-gray text-sm">
               <div className="flex justify-center gap-1.5 mb-3">
@@ -246,7 +325,7 @@ export const ScheduledMessagesPanel = ({ onClose }: Props) => {
             </div>
           ) : messages.length === 0 ? (
             <div className="px-6 py-12 text-center text-warm-gray text-sm">
-              No scheduled messages yet.
+              {listError ? "Scheduled messages could not be loaded." : "No scheduled messages yet."}
             </div>
           ) : (
             <Table>
@@ -287,26 +366,35 @@ export const ScheduledMessagesPanel = ({ onClose }: Props) => {
                       {msg.recurrence === "none" ? "—" : msg.recurrence}
                     </TableCell>
                     <TableCell>
-                      <Badge variant={STATUS_BADGE[msg.status]}>{msg.status}</Badge>
+                      <Badge variant={STATUS_BADGE[msg.status]}>{DELIVERY_STATUS_LABELS[msg.status]}</Badge>
+                      {msg.error_message && <p className="text-xs text-red-500 mt-1 break-words">{msg.error_message}</p>}
+                      {msg.status === "uncertain" && <p className="text-xs text-warm-gray mt-1">{UNCERTAIN_DELIVERY_HELP}</p>}
                     </TableCell>
                     <TableCell>
                       <div className="flex gap-1">
-                        {msg.status === "pending" && (
+                        <Button size="sm" variant="ghost-dark" onClick={() => setHistoryId(msg.id)} aria-label={`Delivery history for ${msg.recipient}`} title="Delivery history">
+                          <Clock size={11} />
+                        </Button>
+                        {(msg.status === "pending" || msg.status === "failed" || (msg.recurrence !== "none" && msg.status !== "cancelled")) && (
                           <Button
                             size="sm"
                             variant="ghost-dark"
-                            onClick={() => handleCancel(msg.id)}
-                            title="Cancel"
+                            onClick={() => void runAction(msg, "cancel")}
+                            disabled={busyId !== null}
+                            aria-label={msg.recurrence === "none" ? "Cancel message" : "Cancel series"}
+                            title={msg.recurrence === "none" ? "Cancel message" : "Cancel series — stop future occurrences; an in-flight delivery may still complete"}
                           >
                             <XCircle size={11} />
                           </Button>
                         )}
-                        {msg.status === "failed" && (
+                        {canRetryScheduledMessage(msg) && (
                           <Button
                             size="sm"
                             variant="ghost-dark"
-                            onClick={() => handleRetry(msg.id)}
-                            title="Retry"
+                            onClick={() => void runAction(msg, "retry")}
+                            disabled={busyId !== null}
+                            aria-label="Retry failed delivery"
+                            title="Retry failed delivery"
                           >
                             <RotateCcw size={11} />
                           </Button>
@@ -315,8 +403,10 @@ export const ScheduledMessagesPanel = ({ onClose }: Props) => {
                           <Button
                             size="sm"
                             variant="destructive"
-                            onClick={() => handleDelete(msg.id)}
-                            title="Delete"
+                            onClick={() => void runAction(msg, "delete")}
+                            disabled={busyId !== null}
+                            aria-label={msg.recurrence === "none" ? "Delete message" : "Delete occurrence and cancel series"}
+                            title={msg.recurrence === "none" ? "Delete message" : "Delete occurrence and cancel series"}
                           >
                             <Trash2 size={11} />
                           </Button>
@@ -328,6 +418,17 @@ export const ScheduledMessagesPanel = ({ onClose }: Props) => {
               </TableBody>
             </Table>
           )}
+          {historyId && (
+            <section aria-label="Delivery attempt history" className="px-6 py-4 border-t border-sand-light space-y-3">
+              <div className="flex items-center justify-between gap-2">
+                <h3 className="text-sm font-semibold">Delivery attempt history</h3>
+                <Button size="sm" variant="ghost-dark" onClick={() => setHistoryVersion((version) => version + 1)} disabled={historyLoading}>Refresh history</Button>
+                <Button size="sm" variant="ghost-dark" onClick={() => setHistoryId(null)}>Close history</Button>
+              </div>
+              <p className="text-xs break-all">Message: {historyId}</p>
+              {historyLoading ? <p role="status" className="text-xs">Loading delivery history...</p> : historyError ? <p role="alert" className="text-xs text-red-500">{historyError}</p> : <DeliveryHistory deliveries={deliveries} />}
+            </section>
+          )}
         </div>
       </div>
     </div>

raggr-frontend/src/components/SidebarContent.tsx

@@ -1,31 +1,41 @@
+import { useEffect, useRef, useState } from "react";
+import { MemoryPanel } from "./MemoryPanel";
+import { IntegrationStatusPanel } from "./IntegrationStatusPanel";
 import { Link } from "react-router-dom";
 import {
-  Clock,
-  LogOut,
-  PanelLeftClose,
-  Shield,
-  ListTree,
-  X,
+	Brain,
+	Clock,
+	LogOut,
+	PanelLeftClose,
+	Shield,
+	ListTree,
+	X,
 } from "lucide-react";
 import { ConversationList } from "./ConversationList";
 import type { Conversation } from "../hooks/useConversations";
 import catIcon from "../assets/cat.png";
 
 type SidebarContentProps = {
-  conversations: Conversation[];
-  onCreateNewConversation: () => void;
-  onSelectConversation: (conversation: Conversation) => void;
-  onRenameConversation: (id: string, title: string) => void;
-  onDeleteConversation: (id: string) => void;
-  selectedId?: string;
-  isAdmin: boolean;
-  onShowAdmin: () => void;
-  onShowScheduler: () => void;
-  onLogout: () => void;
-  /** Desktop: collapse the sidebar. Renders a PanelLeftClose button. */
-  onCollapse?: () => void;
-  /** Mobile: close the drawer. Renders an X button. */
-  onCloseDrawer?: () => void;
+	conversations: Conversation[];
+	onCreateNewConversation: () => void;
+	onSelectConversation: (conversation: Conversation) => void;
+	onRenameConversation: (id: string, title: string) => void;
+	onDeleteConversation: (id: string) => void;
+	onPinConversation?: (id: string, pinned: boolean) => void | Promise<void>;
+	onExportConversation?: (
+		id: string,
+		format: "json" | "markdown",
+	) => void | Promise<void>;
+	onShowMemories?: () => void;
+	selectedId?: string;
+	isAdmin: boolean;
+	onShowAdmin: () => void;
+	onShowScheduler: () => void;
+	onLogout: () => void;
+	/** Desktop: collapse the sidebar. Renders a PanelLeftClose button. */
+	onCollapse?: () => void;
+	/** Mobile: close the drawer. Renders an X button. */
+	onCloseDrawer?: () => void;
 };
 
 /**
@@ -34,95 +44,128 @@ type SidebarContentProps = {
  * actions. Exactly one of `onCollapse`/`onCloseDrawer` is expected per usage.
  */
 export const SidebarContent = ({
-  conversations,
-  onCreateNewConversation,
-  onSelectConversation,
-  onRenameConversation,
-  onDeleteConversation,
-  selectedId,
-  isAdmin,
-  onShowAdmin,
-  onShowScheduler,
-  onLogout,
-  onCollapse,
-  onCloseDrawer,
+	conversations,
+	onCreateNewConversation,
+	onSelectConversation,
+	onRenameConversation,
+	onDeleteConversation,
+	onPinConversation,
+	onExportConversation,
+	onShowMemories,
+	selectedId,
+	isAdmin,
+	onShowAdmin,
+	onShowScheduler,
+	onLogout,
+	onCollapse,
+	onCloseDrawer,
 }: SidebarContentProps) => {
-  return (
-    <div className="flex flex-col h-full">
-      <div className="flex items-center justify-between px-4 py-4 border-b border-white/8">
-        <div className="flex items-center gap-2.5">
-          <img src={catIcon} alt="Simba" className="w-12 h-12" />
-          <h2
-            className="text-lg font-bold text-cream tracking-tight"
-            style={{ fontFamily: "var(--font-display)" }}
-          >
-            asksimba
-          </h2>
-        </div>
-        {onCollapse && (
-          <button
-            onClick={onCollapse}
-            className="w-7 h-7 rounded-lg flex items-center justify-center text-cream/40 hover:text-cream hover:bg-white/10 transition-all cursor-pointer"
-          >
-            <PanelLeftClose size={15} />
-          </button>
-        )}
-        {onCloseDrawer && (
-          <button
-            onClick={onCloseDrawer}
-            className="w-7 h-7 rounded-lg flex items-center justify-center text-cream/40 hover:text-cream hover:bg-white/10 transition-all cursor-pointer"
-          >
-            <X size={16} />
-          </button>
-        )}
-      </div>
+	const [memoriesOpen, setMemoriesOpen] = useState(false);
+	const [integrationsOpen, setIntegrationsOpen] = useState(false);
+	const dialogRef = useRef<HTMLDialogElement>(null);
+	useEffect(() => {
+		if (memoriesOpen) dialogRef.current?.showModal();
+		else dialogRef.current?.close();
+	}, [memoriesOpen]);
+	return (
+		<div className="flex flex-col h-full">
+			{integrationsOpen && <IntegrationStatusPanel onClose={() => setIntegrationsOpen(false)} />}
+			<button className="mx-3 my-2 rounded-lg px-3 py-2 text-left text-sm text-cream hover:bg-white/10" onClick={() => setIntegrationsOpen(true)}>Integrations and data freshness</button>
+			<dialog
+				ref={dialogRef}
+				onCancel={() => setMemoriesOpen(false)}
+				onClose={() => setMemoriesOpen(false)}
+				aria-label="Saved memories"
+				className="m-auto w-full max-w-xl rounded-2xl p-0 backdrop:bg-black/40"
+			>
+				{memoriesOpen && <MemoryPanel onClose={() => setMemoriesOpen(false)} />}
+			</dialog>
+			<div className="flex items-center justify-between px-4 py-4 border-b border-white/8">
+				<div className="flex items-center gap-2.5">
+					<img src={catIcon} alt="Simba" className="w-12 h-12" />
+					<h2
+						className="text-lg font-bold text-cream tracking-tight"
+						style={{ fontFamily: "var(--font-display)" }}
+					>
+						asksimba
+					</h2>
+				</div>
+				{onCollapse && (
+					<button
+						onClick={onCollapse}
+						className="w-7 h-7 rounded-lg flex items-center justify-center text-cream/40 hover:text-cream hover:bg-white/10 transition-all cursor-pointer"
+					>
+						<PanelLeftClose size={15} />
+					</button>
+				)}
+				{onCloseDrawer && (
+					<button
+						onClick={onCloseDrawer}
+						className="w-7 h-7 rounded-lg flex items-center justify-center text-cream/40 hover:text-cream hover:bg-white/10 transition-all cursor-pointer"
+					>
+						<X size={16} />
+					</button>
+				)}
+			</div>
 
-      <div className="flex-1 overflow-y-auto px-2 py-3">
-        <ConversationList
-          conversations={conversations}
-          onCreateNewConversation={onCreateNewConversation}
-          onSelectConversation={onSelectConversation}
-          onRenameConversation={onRenameConversation}
-          onDeleteConversation={onDeleteConversation}
-          selectedId={selectedId}
-        />
-      </div>
+			<div className="flex-1 overflow-y-auto px-2 py-3">
+				<ConversationList
+					conversations={conversations}
+					onCreateNewConversation={onCreateNewConversation}
+					onSelectConversation={onSelectConversation}
+					onRenameConversation={onRenameConversation}
+					onDeleteConversation={onDeleteConversation}
+					onPinConversation={onPinConversation}
+					onExportConversation={onExportConversation}
+					selectedId={selectedId}
+				/>
+			</div>
 
-      <div className="px-2 pb-3 pt-2 border-t border-white/8 flex flex-col gap-0.5">
-        <Link
-          to="/conversations"
-          onClick={onCloseDrawer}
-          className="flex items-center gap-2 w-full px-3 py-2 rounded-xl text-sm text-cream/50 hover:text-cream hover:bg-white/8 transition-all cursor-pointer"
-        >
-          <ListTree size={14} />
-          <span>See all conversations</span>
-        </Link>
-        {isAdmin && (
-          <>
-            <button
-              onClick={onShowAdmin}
-              className="flex items-center gap-2 w-full px-3 py-2 rounded-xl text-sm text-cream/50 hover:text-cream hover:bg-white/8 transition-all cursor-pointer"
-            >
-              <Shield size={14} />
-              <span>Admin</span>
-            </button>
-            <button
-              onClick={onShowScheduler}
-              className="flex items-center gap-2 w-full px-3 py-2 rounded-xl text-sm text-cream/50 hover:text-cream hover:bg-white/8 transition-all cursor-pointer"
-            >
-              <Clock size={14} />
-              <span>Scheduler</span>
-            </button>
-          </>
-        )}
-        <button
-          onClick={onLogout}
-          className="flex items-center gap-2 w-full px-3 py-2 rounded-xl text-sm text-cream/50 hover:text-cream hover:bg-white/8 transition-all cursor-pointer"
-        >
-          <LogOut size={14} />
-          <span>Sign out</span>
-        </button>
-      </div>
-    </div>
-  );
+			<div className="px-2 pb-3 pt-2 border-t border-white/8 flex flex-col gap-0.5">
+				<Link
+					to="/conversations"
+					onClick={onCloseDrawer}
+					className="flex items-center gap-2 w-full px-3 py-2 rounded-xl text-sm text-cream/50 hover:text-cream hover:bg-white/8 transition-all cursor-pointer"
+				>
+					<ListTree size={14} />
+					<span>See all conversations</span>
+				</Link>
+				<button
+					type="button"
+					onClick={() =>
+						onShowMemories ? onShowMemories() : setMemoriesOpen(true)
+					}
+					className="flex items-center gap-2 w-full px-3 py-2 rounded-xl text-sm text-cream/50 hover:text-cream hover:bg-white/8 transition-all cursor-pointer"
+				>
+					<Brain size={14} />
+					<span>Saved memories</span>
+				</button>
+				{isAdmin && (
+					<>
+						<button
+							onClick={onShowAdmin}
+							className="flex items-center gap-2 w-full px-3 py-2 rounded-xl text-sm text-cream/50 hover:text-cream hover:bg-white/8 transition-all cursor-pointer"
+						>
+							<Shield size={14} />
+							<span>Admin</span>
+						</button>
+						<button
+							onClick={onShowScheduler}
+							className="flex items-center gap-2 w-full px-3 py-2 rounded-xl text-sm text-cream/50 hover:text-cream hover:bg-white/8 transition-all cursor-pointer"
+						>
+							<Clock size={14} />
+							<span>Scheduler</span>
+						</button>
+					</>
+				)}
+				<button
+					onClick={onLogout}
+					className="flex items-center gap-2 w-full px-3 py-2 rounded-xl text-sm text-cream/50 hover:text-cream hover:bg-white/8 transition-all cursor-pointer"
+				>
+					<LogOut size={14} />
+					<span>Sign out</span>
+				</button>
+			</div>
+		</div>
+	);
 };

raggr-frontend/src/components/SourceViewer.tsx

@@ -0,0 +1,38 @@
+import { useEffect, useState } from "react";
+import { Link, useParams } from "react-router-dom";
+import ReactMarkdown from "react-markdown";
+import remarkGfm from "remark-gfm";
+import { userService } from "../api/userService";
+
+type Source = { title: string; source: string; content: string; date?: string; headings: string[]; original_url?: string | null };
+export const SourceViewer = () => {
+  const { sourceId } = useParams();
+  const [source, setSource] = useState<Source | null>(null);
+  const [error, setError] = useState("");
+  const [revision, setRevision] = useState(0);
+  useEffect(() => {
+    const controller = new AbortController();
+    setSource(null); setError("");
+    void (async () => {
+      try {
+        const response = await userService.fetchWithAuth(`/api/sources/${encodeURIComponent(sourceId ?? "")}`, { signal: controller.signal });
+        if (!response.ok) throw new Error(response.status === 404 ? "This source is no longer available." : "Unable to load this source.");
+        const result = await response.json();
+        if (!controller.signal.aborted) setSource(result);
+      } catch (err) {
+        if (!controller.signal.aborted) setError(err instanceof Error ? err.message : "Unable to load this source.");
+      }
+    })();
+    return () => controller.abort();
+  }, [sourceId, revision]);
+  return <main className="min-h-screen bg-cream px-5 py-8 text-charcoal"><article className="mx-auto max-w-3xl">
+    <Link to="/" className="text-leaf-dark underline">Back to chat</Link>
+    {error ? <div role="alert" className="mt-6"><p>{error}</p><button className="mt-2 underline" onClick={() => setRevision((r) => r + 1)}>Retry</button></div> : !source ? <p role="status" className="mt-6">Loading source…</p> : <>
+      <h1 className="mt-6 text-2xl font-bold">{source.title}</h1>
+      <p className="mt-2 text-sm text-warm-gray">{source.source}{source.date ? ` · ${source.date}` : ""}{source.headings.length ? ` · ${source.headings.join(" / ")}` : ""}</p>
+      <p className="my-3 text-sm text-warm-gray">This is the indexed section used for retrieval. The original may have changed since indexing.</p>
+      {source.original_url && <a className="inline-block mb-5 text-leaf-dark underline" href={source.original_url} target="_blank" rel="noopener noreferrer">Open original</a>}
+      <div className="markdown-content rounded-2xl border border-sand-light bg-warm-white p-5"><ReactMarkdown remarkPlugins={[remarkGfm]}>{source.content}</ReactMarkdown></div>
+    </>}
+  </article></main>;
+};

raggr-frontend/src/components/artifacts/ArtifactPanel.tsx

@@ -255,6 +255,12 @@ const PanelBody = ({
           >
             {activeEntry.artifact.title}
           </h2>
+          {activeEntry.artifact.source_ref?.stale && (
+            <p role="status" className="mb-3 rounded-xl bg-amber-pale px-3 py-2 text-sm text-amber-dark">
+              Cached budget data may be stale.
+              {activeEntry.artifact.source_ref.as_of && ` Last synchronized: ${new Date(activeEntry.artifact.source_ref.as_of).toLocaleString()}.`}
+            </p>
+          )}
           <ArtifactRenderer artifact={activeEntry.artifact} />
         </article>
       )}

raggr-frontend/src/components/artifacts/renderers/SearchDocuments.tsx

@@ -19,13 +19,14 @@ export const SearchDocuments = ({ data }: RendererProps<SearchDocumentsData>) =>
             return (
               <li key={i} className="rounded-xl border border-sand-light bg-warm-white px-3 py-2.5">
                 <div className="flex items-start justify-between gap-2">
-                  <h4 className="text-sm font-bold leading-snug text-charcoal">{r.title}</h4>
+                  <h4 className="text-sm font-bold leading-snug text-charcoal">{r.url?.startsWith("/sources/") ? <a className="text-leaf-dark underline" href={r.url} target="_blank" rel="noopener noreferrer">{r.title}</a> : r.title}</h4>
                   {r.document_type && (
                     <span className="shrink-0 rounded-full bg-amber-pale px-2 py-0.5 text-[10px] font-semibold text-amber-dark">
                       {r.document_type}
                     </span>
                   )}
                 </div>
+                {r.headings?.length ? <Subtle>{r.headings.join(" / ")}</Subtle> : null}
                 <p className="mt-1 text-[13px] leading-relaxed text-charcoal/80 line-clamp-4">{r.snippet}</p>
                 {(location || r.date) && (
                   <div className="mt-1.5 flex items-center gap-2 text-[11px] text-warm-gray">

raggr-frontend/src/components/artifacts/renderers/types.ts

@@ -49,6 +49,9 @@ export type SearchDocumentsData = {
     filename?: string;
     path?: string;
     document_type?: string;
+    url?: string;
+    source_id?: string;
+    headings?: string[];
     date?: string;
   }[];
 };

raggr-frontend/src/hooks/useConversationBrowser.ts

@@ -1,5 +1,6 @@
 import { useCallback, useEffect, useRef, useState } from "react";
 import { conversationService } from "../api/conversationService";
+import { historyService } from "../api/historyService";
 import type { Conversation } from "./useConversations";
 
 const PAGE_SIZE = 20;
@@ -10,93 +11,145 @@ const PAGE_SIZE = 20;
  * extra item beyond the page size.
  */
 export function useConversationBrowser() {
-  const [items, setItems] = useState<Conversation[]>([]);
-  const [search, setSearchValue] = useState("");
-  const [hasMore, setHasMore] = useState(false);
-  const [loading, setLoading] = useState(false);
+	const [items, setItems] = useState<Conversation[]>([]);
+	const [search, setSearchValue] = useState("");
+	const [hasMore, setHasMore] = useState(false);
+	const [loading, setLoading] = useState(false);
+	const [error, setError] = useState("");
 
-  // Guards against a slow earlier request overwriting a newer one.
-  const requestIdRef = useRef(0);
+	// Guards against a slow earlier request overwriting a newer one.
+	const requestIdRef = useRef(0);
+	const nextOffsetRef = useRef(0);
+	const searchRef = useRef(search);
+	searchRef.current = search;
 
-  const fetchPage = useCallback(
-    async (offset: number, searchTerm: string) => {
-      const requestId = ++requestIdRef.current;
-      setLoading(true);
-      try {
-        const fetched = await conversationService.getAllConversations({
-          limit: PAGE_SIZE + 1,
-          offset,
-          search: searchTerm || undefined,
-        });
-        if (requestId !== requestIdRef.current) return;
+	const fetchPage = useCallback(async (offset: number, searchTerm: string) => {
+		const requestId = ++requestIdRef.current;
+		setLoading(true);
+		setError("");
+		try {
+			const fetched = await historyService.list({
+				limit: PAGE_SIZE + 1,
+				offset,
+				search: searchTerm,
+			});
+			if (requestId !== requestIdRef.current) return;
 
-        const page = fetched
-          .slice(0, PAGE_SIZE)
-          .map((c) => ({ id: c.id, title: c.name }));
-        setHasMore(fetched.length > PAGE_SIZE);
-        setItems((prev) => (offset === 0 ? page : [...prev, ...page]));
-      } catch (err) {
-        if (requestId === requestIdRef.current) {
-          console.error("Failed to load conversations:", err);
-        }
-      } finally {
-        if (requestId === requestIdRef.current) setLoading(false);
-      }
-    },
-    [],
-  );
+			const page = fetched.slice(0, PAGE_SIZE).map((c) => ({
+				id: c.id,
+				title: c.name,
+				pinned: c.pinned,
+				snippet: c.snippet,
+				matchMessageId: c.match_message_id,
+			}));
+			nextOffsetRef.current = offset + page.length;
+			setHasMore(fetched.length > PAGE_SIZE);
+			setItems((prev) =>
+				offset === 0
+					? page
+					: [
+							...prev,
+							...page.filter((c) => !prev.some((old) => old.id === c.id)),
+						],
+			);
+		} catch (err) {
+			if (requestId === requestIdRef.current) {
+				console.error("Failed to load conversations:", err);
+				setError("Unable to load conversations. Please try again.");
+			}
+		} finally {
+			if (requestId === requestIdRef.current) setLoading(false);
+		}
+	}, []);
 
-  // Debounce search changes; reset to the first page on each new term.
-  useEffect(() => {
-    const handle = setTimeout(() => fetchPage(0, search), 250);
-    return () => clearTimeout(handle);
-  }, [search, fetchPage]);
+	// Debounce search changes; reset to the first page on each new term.
+	useEffect(() => {
+		setHasMore(false);
+		setItems([]);
+		const handle = setTimeout(() => fetchPage(0, search), 250);
+		return () => {
+			clearTimeout(handle);
+			++requestIdRef.current;
+		};
+	}, [search, fetchPage]);
 
-  const loadMore = useCallback(() => {
-    if (loading || !hasMore) return;
-    fetchPage(items.length, search);
-  }, [loading, hasMore, items.length, search, fetchPage]);
+	const loadMore = useCallback(() => {
+		if (loading || !hasMore) return;
+		fetchPage(nextOffsetRef.current, search);
+	}, [loading, hasMore, search, fetchPage]);
 
-  const renameConversation = useCallback(
-    async (id: string, title: string) => {
-      const trimmed = title.trim();
-      if (!trimmed) return;
-      const previous = items;
-      setItems((prev) =>
-        prev.map((c) => (c.id === id ? { ...c, title: trimmed } : c)),
-      );
-      try {
-        await conversationService.renameConversation(id, trimmed);
-      } catch (err) {
-        console.error("Failed to rename conversation:", err);
-        setItems(previous);
-      }
-    },
-    [items],
-  );
+	const renameConversation = useCallback(
+		async (id: string, title: string) => {
+			const trimmed = title.trim();
+			if (!trimmed) return;
+			const previous = items;
+			const requestId = requestIdRef.current;
+			setItems((prev) =>
+				prev.map((c) => (c.id === id ? { ...c, title: trimmed } : c)),
+			);
+			try {
+				await conversationService.renameConversation(id, trimmed);
+				await fetchPage(0, searchRef.current);
+			} catch (err) {
+				console.error("Failed to rename conversation:", err);
+				setError("Unable to rename conversation.");
+				if (requestId === requestIdRef.current) setItems(previous);
+			}
+		},
+		[items, fetchPage],
+	);
 
-  const deleteConversation = useCallback(
-    async (id: string) => {
-      const previous = items;
-      setItems((prev) => prev.filter((c) => c.id !== id));
-      try {
-        await conversationService.deleteConversation(id);
-      } catch (err) {
-        console.error("Failed to delete conversation:", err);
-        setItems(previous);
-      }
-    },
-    [items],
-  );
+	const deleteConversation = useCallback(
+		async (id: string) => {
+			const previous = items;
+			const requestId = requestIdRef.current;
+			setItems((prev) => prev.filter((c) => c.id !== id));
+			try {
+				await conversationService.deleteConversation(id);
+				await fetchPage(0, searchRef.current);
+			} catch (err) {
+				console.error("Failed to delete conversation:", err);
+				setError("Unable to delete conversation.");
+				if (requestId === requestIdRef.current) setItems(previous);
+			}
+		},
+		[items, fetchPage],
+	);
 
-  return {
-    items,
-    search,
-    setSearch: setSearchValue,
-    hasMore,
-    loading,
-    loadMore,
-    renameConversation,
-    deleteConversation,
-  };
+	const pinConversation = useCallback(
+		async (id: string, pinned: boolean) => {
+			try {
+				await historyService.pin(id, pinned);
+				await fetchPage(0, searchRef.current);
+			} catch {
+				setError("Unable to change pin. Please try again.");
+			}
+		},
+		[fetchPage],
+	);
+
+	const exportConversation = useCallback(
+		async (id: string, format: "json" | "markdown") => {
+			try {
+				await historyService.export(id, format);
+			} catch {
+				setError("Unable to export conversation. Please try again.");
+			}
+		},
+		[],
+	);
+
+	return {
+		error,
+		pinConversation,
+		exportConversation,
+		items,
+		search,
+		setSearch: setSearchValue,
+		hasMore,
+		loading,
+		loadMore,
+		renameConversation,
+		deleteConversation,
+	};
 }

raggr-frontend/src/hooks/useConversations.ts

@@ -1,10 +1,14 @@
 import { useState, useCallback, useEffect, useMemo } from "react";
 import { useMatch, useNavigate } from "react-router-dom";
+import { historyService } from "../api/historyService";
 import { conversationService } from "../api/conversationService";
 
 export type Conversation = {
-  title: string;
-  id: string;
+	title: string;
+	id: string;
+	pinned?: boolean;
+	snippet?: string;
+	matchMessageId?: string | null;
 };
 
 // The sidebar/drawer only shows the most recent conversations; the full list
@@ -12,90 +16,108 @@ export type Conversation = {
 const SIDEBAR_LIMIT = 10;
 
 export function useConversations() {
-  const navigate = useNavigate();
-  // ChatScreen now renders under a single catch-all route, so useParams no
-  // longer captures the id. Derive it from the URL directly instead.
-  const conversationId = useMatch("/c/:conversationId")?.params.conversationId;
-  const [conversations, setConversations] = useState<Conversation[]>([]);
+	const navigate = useNavigate();
+	// ChatScreen now renders under a single catch-all route, so useParams no
+	// longer captures the id. Derive it from the URL directly instead.
+	const conversationId = useMatch("/c/:conversationId")?.params.conversationId;
+	const [conversations, setConversations] = useState<Conversation[]>([]);
 
-  const refreshConversations = useCallback(async () => {
-    try {
-      const fetched = await conversationService.getAllConversations({
-        limit: SIDEBAR_LIMIT,
-      });
-      setConversations(fetched.map((c) => ({ id: c.id, title: c.name })));
-    } catch (err) {
-      console.error("Failed to load conversations:", err);
-    }
-  }, []);
+	const refreshConversations = useCallback(async () => {
+		try {
+			const fetched = await historyService.list({
+				limit: SIDEBAR_LIMIT,
+			});
+			setConversations(
+				fetched.map((c) => ({ id: c.id, title: c.name, pinned: c.pinned })),
+			);
+		} catch (err) {
+			console.error("Failed to load conversations:", err);
+		}
+	}, []);
 
-  useEffect(() => {
-    refreshConversations();
-  }, [refreshConversations]);
+	useEffect(() => {
+		refreshConversations();
+	}, [refreshConversations]);
 
-  // The active conversation is derived from the URL. Fall back to a minimal
-  // object while the list is still loading (e.g. on a deep-link refresh).
-  const selectedConversation = useMemo<Conversation | null>(() => {
-    if (!conversationId) return null;
-    return (
-      conversations.find((c) => c.id === conversationId) ?? {
-        id: conversationId,
-        title: "",
-      }
-    );
-  }, [conversationId, conversations]);
+	// The active conversation is derived from the URL. Fall back to a minimal
+	// object while the list is still loading (e.g. on a deep-link refresh).
+	const selectedConversation = useMemo<Conversation | null>(() => {
+		if (!conversationId) return null;
+		return (
+			conversations.find((c) => c.id === conversationId) ?? {
+				id: conversationId,
+				title: "",
+			}
+		);
+	}, [conversationId, conversations]);
 
-  const createConversation = useCallback(async (): Promise<Conversation> => {
-    const newConv = await conversationService.createConversation();
-    const conversation = { title: newConv.name, id: newConv.id };
-    setConversations((prev) => [conversation, ...prev]);
-    navigate(`/c/${conversation.id}`);
-    return conversation;
-  }, [navigate]);
+	const createConversation = useCallback(async (): Promise<Conversation> => {
+		const newConv = await conversationService.createConversation();
+		const conversation = { title: newConv.name, id: newConv.id };
+		setConversations((prev) =>
+			[
+				...prev.filter((c) => c.pinned),
+				conversation,
+				...prev.filter((c) => !c.pinned),
+			].slice(0, SIDEBAR_LIMIT),
+		);
+		navigate(`/c/${conversation.id}`);
+		return conversation;
+	}, [navigate]);
 
-  const renameConversation = useCallback(
-    async (id: string, title: string): Promise<void> => {
-      const trimmed = title.trim();
-      if (!trimmed) return;
-      // Optimistic update, roll back on failure. selectedConversation is
-      // derived from `conversations`, so it updates automatically.
-      const previous = conversations;
-      setConversations((prev) =>
-        prev.map((c) => (c.id === id ? { ...c, title: trimmed } : c)),
-      );
-      try {
-        await conversationService.renameConversation(id, trimmed);
-      } catch (err) {
-        console.error("Failed to rename conversation:", err);
-        setConversations(previous);
-      }
-    },
-    [conversations],
-  );
+	const renameConversation = useCallback(
+		async (id: string, title: string): Promise<void> => {
+			const trimmed = title.trim();
+			if (!trimmed) return;
+			// Optimistic update, roll back on failure. selectedConversation is
+			// derived from `conversations`, so it updates automatically.
+			const previous = conversations;
+			setConversations((prev) =>
+				prev.map((c) => (c.id === id ? { ...c, title: trimmed } : c)),
+			);
+			try {
+				await conversationService.renameConversation(id, trimmed);
+			} catch (err) {
+				console.error("Failed to rename conversation:", err);
+				setConversations(previous);
+			}
+		},
+		[conversations],
+	);
 
-  const deleteConversation = useCallback(
-    async (id: string): Promise<void> => {
-      const previous = conversations;
-      const wasActive = conversationId === id;
-      setConversations((prev) => prev.filter((c) => c.id !== id));
-      // Navigating home clears the message view via the route change.
-      if (wasActive) navigate("/");
-      try {
-        await conversationService.deleteConversation(id);
-      } catch (err) {
-        console.error("Failed to delete conversation:", err);
-        setConversations(previous);
-      }
-    },
-    [conversations, conversationId, navigate],
-  );
+	const deleteConversation = useCallback(
+		async (id: string): Promise<void> => {
+			const previous = conversations;
+			const wasActive = conversationId === id;
+			setConversations((prev) => prev.filter((c) => c.id !== id));
+			// Navigating home clears the message view via the route change.
+			if (wasActive) navigate("/");
+			try {
+				await conversationService.deleteConversation(id);
+			} catch (err) {
+				console.error("Failed to delete conversation:", err);
+				setConversations(previous);
+			}
+		},
+		[conversations, conversationId, navigate],
+	);
 
-  return {
-    conversations,
-    selectedConversation,
-    createConversation,
-    renameConversation,
-    deleteConversation,
-    refreshConversations,
-  };
+	const pinConversation = useCallback(
+		async (id: string, pinned: boolean) => {
+			await historyService.pin(id, pinned);
+			await refreshConversations();
+		},
+		[refreshConversations],
+	);
+
+	return {
+		pinConversation,
+		exportConversation: historyService.export,
+		conversations,
+		selectedConversation,
+		createConversation,
+		renameConversation,
+		deleteConversation,
+		refreshConversations,
+	};
 }

raggr-frontend/src/hooks/useScheduledMessages.ts

@@ -1,4 +1,4 @@
-import { useState, useEffect, useCallback } from "react";
+import { useState, useEffect, useCallback, useRef } from "react";
 import {
   scheduledMessageService,
   type ScheduledMessage,
@@ -7,19 +7,34 @@ import {
 export function useScheduledMessages() {
   const [messages, setMessages] = useState<ScheduledMessage[]>([]);
   const [loading, setLoading] = useState(true);
+  const [error, setError] = useState("");
+  const request = useRef(0);
 
-  const refresh = useCallback(() => {
-    setLoading(true);
-    scheduledMessageService
-      .list()
-      .then(setMessages)
-      .catch(() => {})
-      .finally(() => setLoading(false));
+  const refresh = useCallback(async () => {
+    const current = ++request.current;
+    try {
+      const result = await scheduledMessageService.list();
+      if (current === request.current) {
+        setMessages(result);
+        setError("");
+      }
+    } catch (err) {
+      if (current === request.current) {
+        setError(err instanceof Error ? err.message : "Failed to load scheduled messages");
+      }
+    } finally {
+      if (current === request.current) setLoading(false);
+    }
   }, []);
 
   useEffect(() => {
-    refresh();
+    void refresh();
+    const timer = setInterval(() => void refresh(), 10_000);
+    return () => {
+      clearInterval(timer);
+      ++request.current;
+    };
   }, [refresh]);
 
-  return { messages, loading, refresh };
+  return { messages, loading, error, refresh };
 }

raggr-frontend/tests/history-management.test.cjs

@@ -0,0 +1,105 @@
+const fs = require("node:fs");
+const assert = require("node:assert/strict");
+const test = require("node:test");
+const ts = require("typescript");
+const React = require("react");
+const { renderToStaticMarkup } = require("react-dom/server");
+for (const ext of [".ts", ".tsx"]) {
+	require.extensions[ext] = (mod, filename) => {
+		const { outputText } = ts.transpileModule(
+			fs.readFileSync(filename, "utf8"),
+			{
+				compilerOptions: {
+					module: ts.ModuleKind.CommonJS,
+					jsx: ts.JsxEmit.ReactJSX,
+					esModuleInterop: true,
+				},
+			},
+		);
+		mod._compile(outputText, filename);
+	};
+}
+const { historyService } = require("../src/api/historyService.ts");
+const { userService } = require("../src/api/userService.ts");
+const { ConversationList } = require("../src/components/ConversationList.tsx");
+const { MemoryPanel } = require("../src/components/MemoryPanel.tsx");
+
+test("history requests use authenticated fetch, encode search and send explicit pin boolean", async () => {
+	const calls = [];
+	const original = userService.fetchWithAuth;
+	userService.fetchWithAuth = async (...args) => {
+		calls.push(args);
+		return new Response(JSON.stringify([]), { status: 200 });
+	};
+	try {
+		await historyService.list({ search: "cats & dogs", limit: 21, offset: 20 });
+		const query = new URL(calls[0][0], "https://example.test").searchParams;
+		assert.equal(query.get("search"), "cats & dogs");
+		assert.equal(query.get("offset"), "20");
+		await historyService.pin("id", false);
+		assert.equal(calls[1][0], "/api/history/conversations/id/pin");
+		assert.equal(calls[1][1].method, "PATCH");
+		assert.deepEqual(JSON.parse(calls[1][1].body), { pinned: false });
+		await historyService.correctMemory("id", "correct fact");
+		assert.deepEqual(JSON.parse(calls[2][1].body), { content: "correct fact" });
+		userService.fetchWithAuth = async (...args) => {
+			calls.push(args);
+			return new Response(null, { status: 204 });
+		};
+		assert.equal(await historyService.forgetMemory("id"), undefined);
+		assert.equal(calls[3][1].method, "DELETE");
+	} finally {
+		userService.fetchWithAuth = original;
+	}
+});
+
+test("failed exports and memory updates are surfaced", async () => {
+	const original = userService.fetchWithAuth;
+	userService.fetchWithAuth = async () => new Response(null, { status: 404 });
+	try {
+		await assert.rejects(
+			historyService.export("foreign", "json"),
+			/Unable to export/,
+		);
+		await assert.rejects(
+			historyService.correctMemory("foreign", "content"),
+			/Unable to update/,
+		);
+	} finally {
+		userService.fetchWithAuth = original;
+	}
+});
+
+test("conversation results render escaped snippets, pinned status and keyboard reachable actions", () => {
+	const html = renderToStaticMarkup(
+		React.createElement(ConversationList, {
+			conversations: [
+				{
+					id: "owned",
+					title: "Cats",
+					pinned: true,
+					snippet: "<script>cats</script>",
+				},
+			],
+			onSelectConversation() {},
+			onCreateNewConversation() {},
+			onPinConversation() {},
+			onExportConversation() {},
+			variant: "light",
+		}),
+	);
+	assert.match(html, /aria-label="Pinned"/);
+	assert.match(html, /aria-label="Actions for Cats"/);
+	assert.match(html, /&lt;script&gt;cats&lt;\/script&gt;/);
+	assert.doesNotMatch(html, /<script>/);
+});
+
+test("memory panel exposes its purpose and loading status", () => {
+	const html = renderToStaticMarkup(
+		React.createElement(MemoryPanel, { onClose() {} }),
+	);
+	assert.match(html, /aria-label="Saved memories"/);
+	assert.match(html, /Correct facts Simba remembers/);
+	assert.match(html, /role="status"/);
+	assert.match(html, />Close</);
+});

raggr-frontend/tests/job-delivery.test.cjs

@@ -0,0 +1,166 @@
+const fs = require("node:fs");
+const assert = require("node:assert/strict");
+const test = require("node:test");
+const ts = require("typescript");
+const React = require("react");
+const { renderToStaticMarkup } = require("react-dom/server");
+
+// Same renderer harness as history-management.test.cjs; no new packages.
+for (const ext of [".ts", ".tsx"]) {
+  require.extensions[ext] = (mod, filename) => {
+    const { outputText } = ts.transpileModule(fs.readFileSync(filename, "utf8"), {
+      compilerOptions: {
+        target: ts.ScriptTarget.ES2020,
+        module: ts.ModuleKind.CommonJS,
+        jsx: ts.JsxEmit.ReactJSX,
+        esModuleInterop: true,
+      },
+    });
+    mod._compile(outputText, filename);
+  };
+}
+const { userService } = require("../src/api/userService.ts");
+const { scheduledMessageService: service, canRetryScheduledMessage } = require("../src/api/scheduledMessageService.ts");
+const hook = require("../src/hooks/useScheduledMessages.ts");
+const { ScheduledMessagesPanel, DeliveryHistory, DELIVERY_STATUS_LABELS } = require("../src/components/ScheduledMessagesPanel.tsx");
+
+function message(status, retryable = false, recurrence = "none") {
+  return {
+    id: "message-id", series_id: "series-id", retryable, status, recurrence,
+    recipient: "person@example.test", channel: "email", content: "Hello", subject: "Hello",
+    scheduled_at: "2026-10-13T12:00:00Z", use_agent: false, error_message: null,
+    created_at: "2026-10-12T12:00:00Z", updated_at: "2026-10-12T12:00:00Z",
+  };
+}
+function panelHtml(messages, error = "") {
+  const original = hook.useScheduledMessages;
+  hook.useScheduledMessages = () => ({ messages, loading: false, error, refresh: async () => {} });
+  try {
+    return renderToStaticMarkup(React.createElement(ScheduledMessagesPanel, { onClose() {} }));
+  } finally {
+    hook.useScheduledMessages = original;
+  }
+}
+
+test("retry requires both failed status and explicit retryable permission", () => {
+  for (const status of ["pending", "processing", "sent", "uncertain", "cancelled"]) {
+    assert.equal(canRetryScheduledMessage(message(status, true)), false);
+  }
+  assert.equal(canRetryScheduledMessage(message("failed", true)), true);
+  assert.equal(canRetryScheduledMessage(message("failed", false)), false);
+  assert.equal(canRetryScheduledMessage({ status: "failed" }), false);
+});
+
+test("renderer labels all statuses and limits retry/delete actions", () => {
+  for (const status of Object.keys(DELIVERY_STATUS_LABELS).filter((s) => s !== "sending")) {
+    const html = panelHtml([message(status, true)]);
+    assert.ok(html.includes(DELIVERY_STATUS_LABELS[status]));
+    assert.equal(html.includes('aria-label="Retry failed delivery"'), status === "failed");
+    assert.equal(html.includes('aria-label="Delete message"'), ["pending", "cancelled"].includes(status));
+    assert.match(html, /Delivery history for person@example.test/);
+  }
+  assert.doesNotMatch(panelHtml([message("failed", false)]), /aria-label="Retry failed delivery"/);
+});
+
+test("uncertain delivery explains reconciliation before resend", () => {
+  const html = panelHtml([message("uncertain")]);
+  assert.match(html, /Delivery may already have occurred/);
+  assert.match(html, /Reconcile with the provider or recipient before any resend/);
+  assert.doesNotMatch(html, /aria-label="Retry failed delivery"/);
+});
+
+test("recurring actions explain series cancellation and deletion", () => {
+  assert.match(panelHtml([message("pending", false, "daily")]), /Delete occurrence and cancel series/);
+  for (const status of ["pending", "failed", "processing", "sent", "uncertain"]) {
+    assert.match(panelHtml([message(status, false, "weekly")]), /aria-label="Cancel series"/);
+  }
+});
+
+test("list errors are surfaced with a refresh action", () => {
+  const html = panelHtml([], "Permission denied");
+  assert.match(html, /role="alert"/);
+  assert.match(html, /Permission denied/);
+  assert.match(html, /Refresh messages/);
+  assert.doesNotMatch(html, /No scheduled messages yet/);
+});
+
+test("delivery history renders every attempt, timing, status and escaped errors", () => {
+  const html = renderToStaticMarkup(React.createElement(DeliveryHistory, { deliveries: [{
+    id: "delivery", run_id: "run-1", status: "uncertain", error: "provider timeout", retryable: false,
+    attempts: [
+      { id: "a1", status: "failed", error: "<script>rejected</script>", started_at: "2026-10-12T12:00:00Z", finished_at: "2026-10-12T12:01:00Z" },
+      { id: "a2", status: "sending", error: null, started_at: "2026-10-12T12:02:00Z", finished_at: null },
+    ],
+  }] }));
+  assert.match(html, /Run: run-1/);
+  assert.match(html, /Attempt 1: Failed/);
+  assert.match(html, /Attempt 2: Sending/);
+  assert.match(html, /Started:/);
+  assert.match(html, /Finished:/);
+  assert.match(html, /Not finished/);
+  assert.match(html, /provider timeout/);
+  assert.match(html, /&lt;script&gt;rejected&lt;\/script&gt;/);
+  assert.match(html, /uncertain deliveries cannot be retried/);
+  assert.match(renderToStaticMarkup(React.createElement(DeliveryHistory, { deliveries: [] })), /No delivery attempts yet/);
+});
+
+test("service retains CRUD and uses authenticated history GET and explicit retry POST", async () => {
+  const original = userService.fetchWithAuth;
+  const calls = [];
+  userService.fetchWithAuth = async (...args) => {
+    calls.push(args);
+    return new Response(JSON.stringify({ id: "result" }), { status: 200 });
+  };
+  try {
+    await service.list();
+    const create = { recipient: "person", channel: "imessage", content: "Hello", scheduled_at: "2026-10-13T12:00:00Z" };
+    await service.create(create);
+    await service.update("id", { scheduled_at: create.scheduled_at });
+    await service.update("id", { status: "cancelled" });
+    await service.deliveries("id/with space");
+    await service.retry("id/with space");
+    await service.remove("id");
+    assert.equal(calls[0][0], "/api/scheduled-messages/");
+    assert.equal(calls[1][1].method, "POST");
+    assert.deepEqual(JSON.parse(calls[1][1].body), create);
+    assert.equal(calls[2][1].method, "PUT");
+    assert.deepEqual(JSON.parse(calls[3][1].body), { status: "cancelled" });
+    assert.deepEqual(calls[4], ["/api/scheduled-messages/id%2Fwith%20space/deliveries"]);
+    assert.deepEqual(calls[5], ["/api/scheduled-messages/id%2Fwith%20space/retry", { method: "POST" }]);
+    assert.equal(calls[6][1].method, "DELETE");
+    userService.fetchWithAuth = async () => new Response(null, { status: 204 });
+    assert.equal(await service.remove("id"), undefined);
+  } finally {
+    userService.fetchWithAuth = original;
+  }
+});
+
+test("PUT cannot reset a failed delivery to pending even through untyped input", async () => {
+  const original = userService.fetchWithAuth;
+  let called = false;
+  userService.fetchWithAuth = async () => { called = true; throw new Error("Unexpected request"); };
+  try {
+    await assert.rejects(service.update("failed", { status: "pending", scheduled_at: "2026-10-13T12:00:00Z" }), /explicit retry endpoint/);
+    assert.equal(called, false);
+  } finally {
+    userService.fetchWithAuth = original;
+  }
+});
+
+test("all service failures surface backend details and non-JSON fallbacks", async () => {
+  const original = userService.fetchWithAuth;
+  try {
+    for (const invoke of [() => service.list(), () => service.create({}), () => service.update("id", {}), () => service.remove("id"), () => service.deliveries("id"), () => service.retry("id")]) {
+      userService.fetchWithAuth = async () => new Response(JSON.stringify({ error: "Only safe failed deliveries can be retried" }), { status: 409 });
+      await assert.rejects(invoke(), /Only safe failed deliveries/);
+      userService.fetchWithAuth = async () => new Response("Unavailable", { status: 503 });
+      await assert.rejects(invoke(), /Failed to/);
+    }
+    userService.fetchWithAuth = async () => new Response(JSON.stringify({ status: "uncertain", error_message: "Delivery requires reconciliation" }), { status: 409 });
+    await assert.rejects(service.retry("id"), /requires reconciliation/);
+    userService.fetchWithAuth = async () => new Response(JSON.stringify({ status: "processing", error_message: null }), { status: 409 });
+    await assert.rejects(service.retry("id"), /Failed to retry scheduled message \(processing\)/);
+  } finally {
+    userService.fetchWithAuth = original;
+  }
+});

raggr-frontend/tests/runs-resume.test.cjs

@@ -0,0 +1,76 @@
+const fs = require('node:fs');
+const assert = require('node:assert/strict');
+const test = require('node:test');
+const ts = require('typescript');
+require.extensions['.ts'] = (mod, filename) => {
+  const { outputText } = ts.transpileModule(fs.readFileSync(filename, 'utf8'), {
+    compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022, esModuleInterop: true },
+  });
+  mod._compile(outputText, filename);
+};
+const { conversationService } = require('../src/api/conversationService.ts');
+const { userService } = require('../src/api/userService.ts');
+const run = { id: 'run', conversation_id: 'conv', status: 'running', query: 'hi' };
+function stream(chunks) {
+  return new Response(new ReadableStream({ start(controller) {
+    for (const chunk of chunks) controller.enqueue(new TextEncoder().encode(chunk));
+    controller.close();
+  }}));
+}
+
+test('SSE parser accepts split CRLF, ids, comments, UTF8, and DONE; ignores duplicate replay', async () => {
+  const events = [], cursors = [];
+  const response = stream([': ping\r', '\n\r\nid: 1\r\ndata: {"type":"content","delta":"猫"}\r', '\n\r\nid: 1\ndata: {"type":"content","delta":"duplicate"}\n\nid: 2\ndata: [DONE]\n\n']);
+  assert.equal(await conversationService._readSSEStream(response, e => events.push(e), id => cursors.push(id)), true);
+  assert.deepEqual(events, [{ type: 'content', delta: '猫' }]);
+  assert.deepEqual(cursors, [1]);
+});
+
+test('premature EOF is recoverable and resume uses last successfully handled cursor', async () => {
+  const original = userService.fetchWithAuth;
+  const calls = [], events = [], connection = [];
+  userService.fetchWithAuth = async (url) => {
+    calls.push(url);
+    return calls.length === 1 ? stream(['id: 1\ndata: {"type":"content","delta":"first"}\n\n']) :
+      stream(['id: 1\ndata: {"type":"content","delta":"duplicate"}\n\nid: 2\ndata: {"type":"response","message":"answer"}\n\nid: 3\ndata: [DONE]\n\n']);
+  };
+  try {
+    await conversationService.resumeRun(run, e => events.push(e), new AbortController().signal, v => connection.push(v));
+    assert.match(calls[1], /cursor=1$/);
+    assert.deepEqual(events.map(e => e.type), ['content', 'response']);
+    assert.deepEqual(connection, [false, true, false]);
+  } finally { userService.fetchWithAuth = original; }
+});
+
+test('ambiguous creation retry reuses idempotency key; discovery and cancellation are authenticated', async () => {
+  const original = userService.fetchWithAuth;
+  const calls = [];
+  userService.fetchWithAuth = async (url, init) => {
+    calls.push([url, init]);
+    if (calls.length === 1) throw new TypeError('network failed');
+    return new Response(JSON.stringify(url.endsWith('/runs') && init?.method !== 'POST' ? { run } : run));
+  };
+  try {
+    const body = { query: 'hi', conversation_id: 'conv', request_id: 'stable' };
+    await conversationService.createRun(body);
+    assert.equal(calls[0][1].body, calls[1][1].body);
+    assert.equal((await conversationService.discoverRun('conv')).id, 'run');
+    await conversationService.cancelRun(run);
+    assert.equal(calls.at(-1)[0], '/api/conversation/conv/runs/run/cancel');
+    assert.equal(calls.at(-1)[1].method, 'POST');
+  } finally { userService.fetchWithAuth = original; }
+});
+
+test('detach never issues cancellation and terminal authorization failures do not retry', async () => {
+  const original = userService.fetchWithAuth;
+  const calls = [];
+  userService.fetchWithAuth = async (url) => { calls.push(url); return new Response('', { status: 404 }); };
+  try {
+    await assert.rejects(conversationService.resumeRun(run, () => {}, new AbortController().signal), /resume/);
+    assert.equal(calls.length, 1);
+    const controller = new AbortController(); controller.abort();
+    await assert.rejects(conversationService.resumeRun(run, () => {}, controller.signal), { name: 'AbortError' });
+    assert.equal(calls.length, 1);
+    assert.ok(calls.every(url => !url.includes('/cancel')));
+  } finally { userService.fetchWithAuth = original; }
+});

scripts/deploy.sh

@@ -40,8 +40,10 @@ if ! docker compose --profile "$NEW" up -d --wait "raggr-$NEW"; then
 fi
 
 if [ -n "$OLD" ]; then
-    echo "==> raggr-$NEW healthy; stopping raggr-$OLD (30s grace for in-flight streams)..."
-    docker compose --profile "$OLD" stop -t 30 "raggr-$OLD"
+    # Allow application run draining plus cleanup; override for operational needs.
+    STOP_SECONDS=${DEPLOY_STOP_SECONDS:-$(( ${CHAT_RUN_DRAIN_SECONDS:-1800} + 60 ))}
+    echo "==> raggr-$NEW healthy; draining raggr-$OLD (${STOP_SECONDS}s grace)..."
+    docker compose --profile "$OLD" stop -t "$STOP_SECONDS" "raggr-$OLD"
     docker compose --profile "$OLD" rm -f "raggr-$OLD"
 fi
 

scripts/validate_config.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Validate local .env and process settings without importing the application."""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from dotenv import load_dotenv
+
+from config.validation import ConfigurationError, validate_configuration
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--development",
+        action="store_true",
+        help="allow missing secrets and local URLs",
+    )
+    args = parser.parse_args()
+    load_dotenv(Path(__file__).resolve().parents[1] / ".env", override=False)
+    try:
+        validate_configuration(os.environ, production=not args.development)
+    except ConfigurationError as error:
+        print(error, file=sys.stderr)
+        return 1
+    print("Configuration valid.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

startup.sh

@@ -1,4 +1,12 @@
 #!/bin/bash
+set -euo pipefail
+
+# Support the historical misspelling for externally supplied environments.
+if [ -z "${TAVILY_API_KEY:-}" ] && [ -n "${TAVILIY_API_KEY:-}" ]; then
+    export TAVILY_API_KEY="${TAVILIY_API_KEY}"
+fi
+
+python scripts/validate_config.py
 
 echo "Running database migrations..."
 aerich upgrade
@@ -7,33 +15,42 @@ aerich upgrade
 mkdir -p /app/data/obsidian
 
 # Start continuous Obsidian sync if enabled
-if [ "${OBSIDIAN_CONTINUOUS_SYNC}" = "true" ]; then
-    if [ -z "${OBSIDIAN_EMAIL}" ] || [ -z "${OBSIDIAN_PASSWORD}" ] || [ -z "${OBSIDIAN_VAULT_ID}" ]; then
+if [ "${OBSIDIAN_CONTINUOUS_SYNC:-false}" = "true" ]; then
+    if [ -z "${OBSIDIAN_EMAIL:-}" ] || [ -z "${OBSIDIAN_PASSWORD:-}" ] || [ -z "${OBSIDIAN_VAULT_ID:-}" ]; then
         echo "WARNING: OBSIDIAN_EMAIL, OBSIDIAN_PASSWORD, or OBSIDIAN_VAULT_ID not set. Skipping sync."
     else
-        echo "Setting up Obsidian sync..."
-
         VAULT_PATH="${OBSIDIAN_VAULT_PATH:-/app/data/obsidian}"
+        SYNC_MODE="${OBSIDIAN_SYNC_MODE:-bidirectional}"
+        mkdir -p "${VAULT_PATH}"
 
-        # Login and setup sync (foreground, must complete before sync starts)
-        if ob login --email "${OBSIDIAN_EMAIL}" --password "${OBSIDIAN_PASSWORD}" && \
-           ob sync-setup \
-               --vault "${OBSIDIAN_VAULT_ID}" \
-               --path "${VAULT_PATH}" \
-               --password "${OBSIDIAN_E2E_PASSWORD}" \
-               --device-name "${OBSIDIAN_DEVICE_NAME:-simbarag}"; then
-            # Remove stale lock from previous container run
-            rm -rf "${VAULT_PATH}/.obsidian/.sync.lock"
-            # Set sync to pull-only (read-only) mode
-            ob sync-config --mode pull-only --path "${VAULT_PATH}"
-            # Start continuous sync in background
-            echo "Starting Obsidian continuous sync (pull-only)..."
-            ob sync --continuous --path "${VAULT_PATH}" &
-        else
-            echo "WARNING: Obsidian sync setup failed. Continuing without sync."
-        fi
+        # Blue and green share the vault volume. flock on a file in it works
+        # across containers, so this container only sets up and runs sync once
+        # the previous container's sync process has exited. Two clients writing
+        # one vault and its sync state would otherwise race. Setup runs in the
+        # background so the app starts while it waits.
+        (
+            flock 9
+            echo "Setting up Obsidian sync (${SYNC_MODE})..."
+            if ob login --email "${OBSIDIAN_EMAIL}" --password "${OBSIDIAN_PASSWORD}" && \
+               ob sync-setup \
+                   --vault "${OBSIDIAN_VAULT_ID}" \
+                   --path "${VAULT_PATH}" \
+                   --password "${OBSIDIAN_E2E_PASSWORD:-}" \
+                   --device-name "${OBSIDIAN_DEVICE_NAME:-simbarag}"; then
+                # Safe now: the lock means no other container's sync is running.
+                rm -rf "${VAULT_PATH}/.obsidian/.sync.lock"
+                if ob sync-config --mode "${SYNC_MODE}" --conflict-strategy merge \
+                    --path "${VAULT_PATH}"; then
+                    echo "Starting Obsidian continuous sync (${SYNC_MODE})..."
+                    exec ob sync --continuous --path "${VAULT_PATH}"
+                fi
+                echo "WARNING: Obsidian sync-config failed. Continuing without sync."
+            else
+                echo "WARNING: Obsidian sync setup failed. Continuing without sync."
+            fi
+        ) 9>"${VAULT_PATH}/.raggr-sync.lock" &
     fi
 fi
 
 echo "Starting application..."
-python app.py
+exec python app.py

tests/unit/test_agent_runtime.py

@@ -0,0 +1,137 @@
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from langchain_core.messages import AIMessage
+from quart import Quart
+
+from blueprints.conversation import runtime
+from metrics import AGENT_REQUESTS_TOTAL
+
+
+@pytest.mark.parametrize(
+    "channel", ["web", "email", "imessage", "whatsapp", "scheduled", "cron"]
+)
+@pytest.mark.parametrize("admin", [False, True])
+async def test_channels_share_identity_memories_permissions_and_summary(
+    monkeypatch, channel, admin
+):
+    app = Quart(__name__)
+    user = SimpleNamespace(id="owner", is_admin=lambda: admin)
+    app.main_agent = SimpleNamespace(
+        ainvoke=AsyncMock(return_value={"messages": [AIMessage(content="answer")]}),
+        aget_state=AsyncMock(return_value=SimpleNamespace(values={"messages": []})),
+    )
+    app.admin_agent = SimpleNamespace(
+        ainvoke=AsyncMock(
+            return_value={"messages": [AIMessage(content="admin answer")]}
+        ),
+        aget_state=AsyncMock(return_value=SimpleNamespace(values={"messages": []})),
+    )
+    summary = AIMessage(content="Saved constraints")
+    app.conversation_summary_store = SimpleNamespace(
+        update=AsyncMock(return_value=SimpleNamespace(as_message=lambda: summary))
+    )
+    prompt = AsyncMock(return_value="Remember the budget")
+    monkeypatch.setattr(runtime, "build_system_prompt_with_memories", prompt)
+    before = AGENT_REQUESTS_TOTAL.labels(channel=channel, result="success")._value.get()
+    async with app.app_context():
+        response = await runtime.invoke_agent_for_user(
+            user, "What is my budget?", "thread", channel=channel
+        )
+    agent = app.admin_agent if admin else app.main_agent
+    assert response["messages"][-1].content == ("admin answer" if admin else "answer")
+    config = agent.ainvoke.call_args.kwargs["config"]["configurable"]
+    assert config["user_id"] == "owner"
+    assert config["is_admin"] is admin
+    assert config["thread_id"] == "thread"
+    assert config["channel"] == channel
+    assert config["system_prompt"] == "Remember the budget"
+    assert config["conversation_summary"] is summary
+    assert config["request_id"]
+    prompt.assert_awaited_once_with("owner", query="What is my budget?")
+    assert (
+        AGENT_REQUESTS_TOTAL.labels(channel=channel, result="success")._value.get()
+        == before + 1
+    )
+
+
+async def test_anonymous_invocation_is_rejected():
+    with pytest.raises(ValueError, match="authenticated user"):
+        await runtime.invoke_agent_for_user(None, "hello", "thread")
+
+
+@pytest.mark.parametrize(
+    "error, outcome",
+    [(RuntimeError("failed"), "error"), (asyncio.CancelledError(), "cancelled")],
+)
+async def test_shared_invocation_measures_failed_and_cancelled_generation(
+    error, outcome
+):
+    agent = SimpleNamespace(ainvoke=AsyncMock(side_effect=error))
+    turn = runtime.PreparedTurn(
+        agent,
+        {"messages": []},
+        {
+            "configurable": {
+                "channel": "web",
+                "request_id": "test",
+                "thread_id": "t",
+                "user_id": "u",
+            }
+        },
+    )
+    before = AGENT_REQUESTS_TOTAL.labels(channel="web", result=outcome)._value.get()
+    with pytest.raises(type(error)):
+        await runtime.invoke_prepared(turn)
+    assert (
+        AGENT_REQUESTS_TOTAL.labels(channel="web", result=outcome)._value.get()
+        == before + 1
+    )
+
+
+async def test_concurrent_turns_on_one_thread_keep_every_exchange(
+    monkeypatch, tmp_path
+):
+    import aiosqlite
+    from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
+    from langgraph.graph import END, START, MessagesState, StateGraph
+
+    monkeypatch.setattr(
+        runtime, "build_system_prompt_with_memories", AsyncMock(return_value="sys")
+    )
+    async with aiosqlite.connect(tmp_path / "checkpoints.db") as conn:
+        saver = AsyncSqliteSaver(conn)
+        await saver.setup()
+
+        async def answer(state):
+            # Yield so an unserialized second turn would read stale history.
+            await asyncio.sleep(0.05)
+            return {
+                "messages": [
+                    AIMessage(content="reply " + state["messages"][-1].content)
+                ]
+            }
+
+        graph = StateGraph(MessagesState)
+        graph.add_node("answer", answer)
+        graph.add_edge(START, "answer")
+        graph.add_edge("answer", END)
+        agent = graph.compile(checkpointer=saver)
+
+        app = Quart(__name__)
+        app.main_agent = app.admin_agent = agent
+        user = SimpleNamespace(id="owner", is_admin=lambda: False)
+        async with app.app_context():
+            await asyncio.gather(
+                *[
+                    runtime.invoke_agent_for_user(user, q, "same-thread")
+                    for q in ("first", "second")
+                ]
+            )
+            state = await agent.aget_state(
+                {"configurable": {"thread_id": "same-thread"}}
+            )
+    contents = [m.content for m in state.values["messages"]]
+    assert sorted(contents) == ["first", "reply first", "reply second", "second"]

tests/unit/test_artifacts.py

@@ -23,6 +23,7 @@ async def db():
             "models": [
                 "blueprints.users.models",
                 "blueprints.conversation.models",
+                "blueprints.conversation.run_models",
                 "blueprints.artifacts.models",
             ]
         },
@@ -84,6 +85,29 @@ class TestCreateArtifact:
         assert result is None
         assert await Artifact.all().count() == 0
 
+    async def test_drops_model_context_before_persisting(self, owner):
+        """model_context carries the full chunk text for the model's own view;
+        it must never reach the DB (or, from there, the UI)."""
+        user, conversation, message = owner
+        artifact = await create_artifact_from_output(
+            output=_output(
+                {
+                    "kind": "search.documents",
+                    "title": "Search: vet",
+                    "data": {"results": [{"snippet": "short"}]},
+                    "model_context": {"results": [{"content": "x" * 5000}]},
+                }
+            ),
+            tool="simba_search",
+            user=user,
+            conversation=conversation,
+            message=message,
+        )
+        assert artifact is not None
+        saved = await Artifact.get(id=artifact.id)
+        assert saved.data == {"results": [{"snippet": "short"}]}
+        assert "model_context" not in saved.data
+
     async def test_plain_string_output(self, owner):
         user, conversation, message = owner
         result = await create_artifact_from_output(
@@ -343,12 +367,12 @@ class TestStreamQuery:
                 "data": {"output": {"messages": [AIMessage(content="Done")]}},
             },
         ]
-        monkeypatch.setattr(conv, "get_agent_for", lambda u: _FakeAgent(events))
+        monkeypatch.setattr(conv.runtime, "get_agent_for", lambda u: _FakeAgent(events))
 
         async def _prompt(user_id, query=""):
             return "system"
 
-        monkeypatch.setattr(conv, "build_system_prompt_with_memories", _prompt)
+        monkeypatch.setattr(conv.runtime, "build_system_prompt_with_memories", _prompt)
 
         app = Quart(__name__)
         app.config["JWT_SECRET_KEY"] = "test-secret"

tests/unit/test_chat_runs.py

@@ -0,0 +1,482 @@
+import asyncio
+import uuid
+from datetime import timedelta
+
+import pytest
+import pytest_asyncio
+from quart import Quart
+from tortoise import Tortoise
+
+from blueprints.conversation import runs
+from blueprints.conversation.models import Conversation, ConversationMessage
+from blueprints.conversation.run_models import ChatRun, ChatRunEvent
+from blueprints.users.models import User
+
+
+@pytest_asyncio.fixture
+async def db(tmp_path):
+    await Tortoise.init(
+        db_url=f"sqlite://{tmp_path / 'runs.sqlite'}",
+        modules={
+            "models": [
+                "blueprints.conversation.models",
+                "blueprints.conversation.run_models",
+                "blueprints.users.models",
+            ]
+        },
+    )
+    await Tortoise.generate_schemas()
+    app = Quart(__name__)
+    app.config["RUN_TEST_DB_URL"] = f"sqlite://{tmp_path / 'runs.sqlite'}"
+    user = await User.create(username="owner", email="owner@test")
+    other = await User.create(username="other", email="other@test")
+    conv = await Conversation.create(name="Chat", user=user)
+    yield app, user, other, conv
+    await runs.drain_runs(app, 0)
+    runs._draining_apps.discard(id(app))
+    await Tortoise.close_connections()
+
+
+async def reserved(user, conv, **kwargs):
+    return await ChatRun.create(
+        owner=user,
+        conversation=conv,
+        request_key=str(uuid.uuid4()),
+        payload={"query": "hello"},
+        active_key=str(conv.id),
+        worker_token=uuid.uuid4(),
+        lease_until=runs.now() + timedelta(seconds=30),
+        **kwargs,
+    )
+
+
+async def wait_terminal(run):
+    for _ in range(200):
+        run = await ChatRun.get(id=run.id)
+        if run.status != "running":
+            return run
+        await asyncio.sleep(0.01)
+    pytest.fail("run did not terminate")
+
+
+async def test_order_cursor_isolation_and_partial_persisted_once(db):
+    app, user, other, conv = db
+    run = await reserved(user, conv)
+    await runs.append(run.id, run.worker_token, {"type": "content", "delta": "one"})
+    await runs.append(run.id, run.worker_token, {"type": "content", "delta": "two"})
+    await runs.finish(run.id, "completed")
+    await runs.finish(run.id, "cancelled")
+    events = await ChatRunEvent.filter(run=run).order_by("sequence")
+    assert [e.sequence for e in events] == [1, 2, 3]
+    assert (
+        await ConversationMessage.filter(conversation=conv, speaker="simba").count()
+        == 1
+    )
+    assert (await ConversationMessage.get(conversation=conv)).text == "onetwo"
+    assert await runs.owned_run(other.id, conv.id, run.id) is None
+    foreign = await Conversation.create(name="other", user=user)
+    assert await runs.owned_run(user.id, foreign.id, run.id) is None
+    async with app.app_context():
+        response = await runs.replay_response(run, 1)
+        text = (await response.get_data()).decode()
+    assert "id: 2" in text and "id: 1" not in text
+    assert text.index("id: 2") < text.index("[DONE]")
+
+
+async def test_atomic_idempotency_and_concurrent_slot(db, monkeypatch):
+    app, user, other, conv = db
+    gate = asyncio.Event()
+
+    async def fake(*args):
+        await gate.wait()
+
+    monkeypatch.setattr(runs, "_work", fake)
+    data = {"conversation_id": str(conv.id), "query": "hi", "request_id": "same"}
+    results = await asyncio.gather(
+        *(runs.create_run(app, user.id, data) for _ in range(4))
+    )
+    assert len({str(run.id) for run, _ in results}) == 1
+    assert sum(created for _, created in results) == 1
+    assert await ChatRun.all().count() == 1
+    with pytest.raises(runs.RunConflict, match="different input"):
+        await runs.create_run(app, user.id, {**data, "query": "changed"})
+    with pytest.raises(runs.RunConflict, match="already active"):
+        await runs.create_run(app, user.id, {**data, "request_id": "another"})
+    with pytest.raises(runs.RunConflict) as err:
+        await runs.create_run(app, other.id, data)
+    assert err.value.status == 404
+    gate.set()
+    await asyncio.sleep(0)
+
+
+async def test_expired_restart_never_regenerates_and_releases_slot(db):
+    app, user, _, conv = db
+    run = await reserved(user, conv)
+    await runs.append(run.id, run.worker_token, {"type": "content", "delta": "partial"})
+    await ChatRun.filter(id=run.id).update(
+        lease_until=runs.now() - timedelta(seconds=1)
+    )
+    # Close/reopen the real file database, discarding process-local ORM state.
+    await Tortoise.close_connections()
+    await Tortoise.init(
+        db_url=app.config["RUN_TEST_DB_URL"],
+        modules={
+            "models": [
+                "blueprints.conversation.models",
+                "blueprints.conversation.run_models",
+                "blueprints.users.models",
+            ]
+        },
+    )
+    await runs.expire_runs(str(conv.id))
+    await runs.expire_runs(str(conv.id))
+    latest = await ChatRun.get(id=run.id)
+    assert latest.status == "interrupted" and latest.active_key is None
+    assert await ConversationMessage.filter(conversation=conv).count() == 1
+    assert (await ChatRunEvent.filter(run=run).order_by("sequence"))[1].payload[
+        "type"
+    ] == "error"
+    with pytest.raises(asyncio.CancelledError):
+        await runs.append(
+            run.id, run.worker_token, {"type": "content", "delta": "zombie"}
+        )
+
+
+async def test_background_context_detach_and_cross_process_cancel(db, monkeypatch):
+    app, user, _, conv = db
+    gate = asyncio.Event()
+    started = asyncio.Event()
+    from quart import current_app
+    import blueprints.conversation as conversation
+
+    async def fake(data, user, conv, save_user, save_answer):
+        assert current_app._get_current_object() is app
+        await save_user("hi", None)
+        yield 'data: {"type":"content","delta":"partial"}\n\n'
+        started.set()
+        await gate.wait()
+        await save_answer("complete")
+        yield 'data: {"type":"response","message":"complete"}\n\n'
+        yield "data: [DONE]\n\n"
+
+    monkeypatch.setattr(conversation, "_chat_run_stream", fake)
+    run, _ = await runs.create_run(
+        app,
+        user.id,
+        {"conversation_id": str(conv.id), "query": "hi", "request_id": "turn"},
+    )
+    await started.wait()
+    async with app.app_context():
+        response = await runs.replay_response(run, 0)
+        subscriber = response.response.__aiter__()
+        frame = ""
+        while "partial" not in frame:  # skip keepalives until the delta is written
+            frame = await asyncio.wait_for(subscriber.__anext__(), 2)
+        await subscriber.aclose()  # browser disconnect
+    assert (await ChatRun.get(id=run.id)).status == "running"
+    # Same write made by another process, no local task cancellation API involved.
+    await ChatRun.filter(id=run.id).update(cancel_requested=True)
+    latest = await wait_terminal(run)
+    assert latest.status == "cancelled"
+    assert (
+        await ConversationMessage.filter(conversation=conv, speaker="user").count() == 1
+    )
+    assert (
+        await ConversationMessage.filter(conversation=conv, speaker="simba").count()
+        == 1
+    )
+    assert (
+        await ConversationMessage.get(conversation=conv, speaker="simba")
+    ).text == "partial"
+
+
+async def test_completed_answer_once_and_drain(db, monkeypatch):
+    app, user, _, conv = db
+    import blueprints.conversation as conversation
+
+    async def fake(data, user, conv, save_user, save_answer):
+        await save_user("hi", None)
+        yield 'data: {"type":"content","delta":"answer"}\n\n'
+        await save_answer("answer")
+        yield 'data: {"type":"response","message":"answer"}\n\n'
+
+    monkeypatch.setattr(conversation, "_chat_run_stream", fake)
+    run, _ = await runs.create_run(
+        app,
+        user.id,
+        {"conversation_id": str(conv.id), "query": "hi", "request_id": "turn"},
+    )
+    await runs.drain_runs(app, 2)
+    latest = await ChatRun.get(id=run.id)
+    assert latest.status == "completed"
+    assert (
+        await ConversationMessage.filter(conversation=conv, speaker="simba").count()
+        == 1
+    )
+    with pytest.raises(runs.RunConflict) as err:
+        await runs.create_run(
+            app, user.id, {"conversation_id": str(conv.id), "query": "new"}
+        )
+    assert err.value.status == 503
+
+
+async def test_authenticated_routes_validate_owner_conversation_cursor_and_cancel(db):
+    from quart_jwt_extended import JWTManager, create_access_token
+
+    app, owner, foreign, conv = db
+    app.config.update(
+        TESTING=True, JWT_SECRET_KEY="chat-runs-test-secret-at-least-32-characters"
+    )
+    JWTManager(app)
+    app.register_blueprint(runs.runs_blueprint)
+    async with app.app_context():
+        headers = {
+            "Authorization": "Bearer " + create_access_token(identity=str(owner.id))
+        }
+        other_headers = {
+            "Authorization": "Bearer " + create_access_token(identity=str(foreign.id))
+        }
+    client = app.test_client()
+    run = await reserved(owner, conv)
+    base = f"/api/conversation/{conv.id}/runs/{run.id}"
+    assert (await client.get(base + "/events")).status_code == 401
+    assert (
+        await client.get(base + "/events", headers=other_headers)
+    ).status_code == 404
+    assert (
+        await client.post(base + "/cancel", headers=other_headers)
+    ).status_code == 404
+    assert not (await ChatRun.get(id=run.id)).cancel_requested
+    for cursor in ("-1", "bad", "999"):
+        assert (
+            await client.get(base + f"/events?cursor={cursor}", headers=headers)
+        ).status_code == 400
+    assert (await client.post(base + "/cancel", headers=headers)).status_code == 200
+    assert (await ChatRun.get(id=run.id)).cancel_requested
+    assert (
+        await client.get(f"/api/conversation/{conv.id}/runs", headers=other_headers)
+    ).status_code == 404
+    discovery = await (
+        await client.get(f"/api/conversation/{conv.id}/runs", headers=headers)
+    ).get_json()
+    assert discovery["run"]["id"] == str(run.id)
+    assert (
+        await client.post(
+            "/api/conversation/runs",
+            headers=other_headers,
+            json={"conversation_id": str(conv.id), "query": "hi"},
+        )
+    ).status_code == 404
+    assert (
+        await client.post("/api/conversation/runs", headers=headers, json=[])
+    ).status_code == 400
+
+
+async def test_real_shared_runtime_legacy_endpoint_persists_and_replays(
+    db, monkeypatch
+):
+    from quart_jwt_extended import JWTManager, create_access_token
+    from langchain_core.messages import AIMessageChunk, AIMessage
+    import blueprints.conversation as conversation
+
+    app, owner, _, conv = db
+
+    class Agent:
+        async def astream_events(self, payload, version, config):
+            assert config["configurable"]["user_id"] == str(owner.id)
+            yield {"event": "on_chat_model_start", "run_id": "llm", "name": "model"}
+            yield {
+                "event": "on_chat_model_stream",
+                "run_id": "llm",
+                "data": {"chunk": AIMessageChunk(content="answer")},
+            }
+            yield {
+                "event": "on_chat_model_end",
+                "run_id": "llm",
+                "data": {"output": AIMessage(content="answer")},
+            }
+            yield {
+                "event": "on_chain_end",
+                "data": {"output": {"messages": [AIMessage(content="answer")]}},
+            }
+
+    app.main_agent = Agent()
+    from unittest.mock import AsyncMock
+
+    monkeypatch.setattr(
+        conversation.runtime,
+        "build_system_prompt_with_memories",
+        AsyncMock(return_value="system"),
+    )
+    app.config.update(
+        TESTING=True, JWT_SECRET_KEY="chat-runs-test-secret-at-least-32-characters"
+    )
+    JWTManager(app)
+    app.register_blueprint(conversation.conversation_blueprint)
+    app.register_blueprint(runs.runs_blueprint)
+    async with app.app_context():
+        headers = {
+            "Authorization": "Bearer " + create_access_token(identity=str(owner.id))
+        }
+    response = await app.test_client().post(
+        "/api/conversation/stream-query",
+        headers=headers,
+        json={"conversation_id": str(conv.id), "query": "hi", "request_id": "legacy"},
+    )
+    body = (await response.get_data()).decode()
+    assert (
+        response.status_code == 200
+        and '"type": "response"' in body
+        and "[DONE]" in body
+    )
+    run = await ChatRun.get(request_key="legacy")
+    assert run.status == "completed"
+    assert (
+        await ConversationMessage.filter(conversation=conv, speaker="simba").count()
+        == 1
+    )
+    replay = await app.test_client().get(
+        f"/api/conversation/{conv.id}/runs/{run.id}/events?cursor=0", headers=headers
+    )
+    assert (await replay.get_data()).decode() == body.replace(": keepalive\n\n", "")
+
+
+async def test_nonstream_query_validates_input_and_respects_active_run(db):
+    from quart_jwt_extended import JWTManager, create_access_token
+
+    from blueprints.conversation import conversation_blueprint
+
+    app, owner, _, conv = db
+    app.config.update(
+        TESTING=True, JWT_SECRET_KEY="chat-runs-test-secret-at-least-32-characters"
+    )
+    JWTManager(app)
+    app.register_blueprint(conversation_blueprint)
+    async with app.app_context():
+        headers = {
+            "Authorization": "Bearer " + create_access_token(identity=str(owner.id))
+        }
+    client = app.test_client()
+    for body in (
+        ["not", "an", "object"],
+        {"conversation_id": str(conv.id), "query": 123},
+        {"conversation_id": str(conv.id), "query": "   "},
+    ):
+        resp = await client.post("/api/conversation/query", headers=headers, json=body)
+        assert resp.status_code == 400, body
+    resp = await client.post(
+        "/api/conversation/query",
+        headers=headers,
+        json={"conversation_id": ["x"], "query": "hi"},
+    )
+    assert resp.status_code == 404
+
+    # A durable run owns the checkpoint thread: reject before saving anything.
+    await reserved(owner, conv)
+    resp = await client.post(
+        "/api/conversation/query",
+        headers=headers,
+        json={"conversation_id": str(conv.id), "query": "hi"},
+    )
+    assert resp.status_code == 409
+    assert await ConversationMessage.filter(conversation=conv).count() == 0
+
+
+async def test_deltas_coalesce_and_local_subscribers_wake_immediately(db, monkeypatch):
+    app, user, _, conv = db
+    import blueprints.conversation as conversation
+
+    release = asyncio.Event()
+
+    async def fake(data, user, conv, save_user, save_answer):
+        await save_user("hi", None)
+        for word in ("one ", "two ", "three"):
+            yield f'data: {{"type":"content","delta":"{word}"}}\n\n'
+        yield 'data: {"type":"tool_start","tool":"t"}\n\n'
+        await release.wait()
+        await save_answer("one two three")
+        yield 'data: {"type":"response","message":"one two three"}\n\n'
+
+    monkeypatch.setattr(conversation, "_chat_run_stream", fake)
+    monkeypatch.setattr(runs, "POLL_SECONDS", 30)  # only a wakeup can be fast
+    run, _ = await runs.create_run(
+        app, user.id, {"conversation_id": str(conv.id), "query": "hi"}
+    )
+    async with app.app_context():
+        response = await runs.replay_response(run, 0)
+        frames = response.response.__aiter__()
+        seen = []
+        while not any('"tool_start"' in f for f in seen):
+            seen.append(await asyncio.wait_for(frames.__anext__(), 2))
+        release.set()
+        while "[DONE]" not in seen[-1]:
+            seen.append(await asyncio.wait_for(frames.__anext__(), 2))
+    deltas = [
+        e.payload["delta"]
+        for e in await ChatRunEvent.filter(run=run).order_by("sequence")
+        if e.payload["type"] == "content"
+    ]
+    # First delta is written immediately; the burst behind it shares one write.
+    assert "".join(deltas) == "one two three"
+    assert len(deltas) == 2
+    await wait_terminal(run)
+
+
+async def _content_deltas(run):
+    return [
+        e.payload["delta"]
+        for e in await ChatRunEvent.filter(run=run).order_by("sequence")
+        if e.payload["type"] == "content"
+    ]
+
+
+async def test_buffered_delta_flushes_while_model_is_silent(db, monkeypatch):
+    app, user, _, conv = db
+    import blueprints.conversation as conversation
+
+    gate = asyncio.Event()
+
+    async def fake(data, user, conv, save_user, save_answer):
+        await save_user("hi", None)
+        yield 'data: {"type":"content","delta":"a"}\n\n'
+        yield 'data: {"type":"content","delta":"b"}\n\n'
+        await gate.wait()  # e.g. the model is writing tool-call arguments
+        yield 'data: {"type":"response","message":"ab"}\n\n'
+
+    monkeypatch.setattr(conversation, "_chat_run_stream", fake)
+    monkeypatch.setattr(runs, "FLUSH_SECONDS", 0.05)
+    run, _ = await runs.create_run(
+        app, user.id, {"conversation_id": str(conv.id), "query": "hi"}
+    )
+    for _ in range(100):
+        if "".join(await _content_deltas(run)) == "ab":
+            break
+        await asyncio.sleep(0.01)
+    assert "".join(await _content_deltas(run)) == "ab"
+    gate.set()
+    await wait_terminal(run)
+
+
+async def test_stop_keeps_buffered_text_in_saved_answer(db, monkeypatch):
+    app, user, _, conv = db
+    import blueprints.conversation as conversation
+
+    buffered = asyncio.Event()
+
+    async def fake(data, user, conv, save_user, save_answer):
+        await save_user("hi", None)
+        yield 'data: {"type":"content","delta":"first "}\n\n'
+        yield 'data: {"type":"content","delta":"tail"}\n\n'
+        buffered.set()
+        await asyncio.Event().wait()
+
+    monkeypatch.setattr(conversation, "_chat_run_stream", fake)
+    monkeypatch.setattr(runs, "FLUSH_SECONDS", 60)  # "tail" stays buffered
+    run, _ = await runs.create_run(
+        app, user.id, {"conversation_id": str(conv.id), "query": "hi"}
+    )
+    await buffered.wait()
+    await ChatRun.filter(id=run.id).update(cancel_requested=True)
+    assert (await wait_terminal(run)).status == "cancelled"
+    answer = await ConversationMessage.get(conversation=conv, speaker="simba")
+    assert answer.text == "first tail"

tests/unit/test_checkpoints.py

@@ -0,0 +1,226 @@
+"""LangGraph checkpoint pruning/deletion: bounds the on-disk checkpoint DB."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import aiosqlite
+import pytest
+from langchain_core.messages import AIMessage, HumanMessage
+from langgraph.checkpoint.base import empty_checkpoint
+from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
+from langgraph.graph import END, START, MessagesState, StateGraph
+from quart import Quart
+from quart_jwt_extended import JWTManager, create_access_token
+from tortoise import Tortoise
+
+from blueprints.conversation import conversation_blueprint
+from blueprints.conversation.checkpoints import (
+    DEFAULT_KEEP_PER_THREAD,
+    delete_thread_checkpoints,
+    keep_per_thread,
+    prune_thread_checkpoints,
+)
+from blueprints.conversation.models import Conversation
+from blueprints.users.models import User
+
+
+async def _checkpoint_ids(saver, thread_id):
+    async with saver.conn.execute(
+        "SELECT checkpoint_id FROM checkpoints WHERE thread_id = ? ORDER BY checkpoint_id",
+        (str(thread_id),),
+    ) as cur:
+        return [row[0] async for row in cur]
+
+
+async def _write_rows(saver, thread_id):
+    async with saver.conn.execute(
+        "SELECT checkpoint_id FROM writes WHERE thread_id = ?",
+        (str(thread_id),),
+    ) as cur:
+        return [row[0] async for row in cur]
+
+
+async def _add_checkpoint(saver, thread_id):
+    checkpoint = empty_checkpoint()
+    config = {"configurable": {"thread_id": str(thread_id), "checkpoint_ns": ""}}
+    await saver.aput(config, checkpoint, {}, {})
+    return checkpoint["id"]
+
+
+@pytest.fixture
+async def saver(tmp_path):
+    async with aiosqlite.connect(tmp_path / "checkpoints.db") as conn:
+        s = AsyncSqliteSaver(conn)
+        await s.setup()
+        yield s
+
+
+async def test_prune_keeps_newest_n_and_drops_orphaned_writes_only_for_pruned_thread(
+    saver,
+):
+    kept_ids = []
+    pruned_ids = []
+    for i in range(5):
+        cid = await _add_checkpoint(saver, "pruned-thread")
+        (pruned_ids if i < 3 else kept_ids).append(cid)
+        await saver.aput_writes(
+            {
+                "configurable": {
+                    "thread_id": "pruned-thread",
+                    "checkpoint_ns": "",
+                    "checkpoint_id": cid,
+                }
+            },
+            [("some_channel", "value")],
+            f"task-{i}",
+        )
+    other_id = await _add_checkpoint(saver, "other-thread")
+    await saver.aput_writes(
+        {
+            "configurable": {
+                "thread_id": "other-thread",
+                "checkpoint_ns": "",
+                "checkpoint_id": other_id,
+            }
+        },
+        [("some_channel", "value")],
+        "task-other",
+    )
+
+    await prune_thread_checkpoints(saver, "pruned-thread", keep=2)
+
+    remaining = await _checkpoint_ids(saver, "pruned-thread")
+    assert sorted(remaining) == sorted(kept_ids)
+    remaining_writes = await _write_rows(saver, "pruned-thread")
+    assert sorted(remaining_writes) == sorted(kept_ids)
+    for cid in pruned_ids:
+        assert cid not in remaining
+        assert cid not in remaining_writes
+
+    # Untouched: pruning one thread must not affect another.
+    assert await _checkpoint_ids(saver, "other-thread") == [other_id]
+    assert await _write_rows(saver, "other-thread") == [other_id]
+
+
+async def test_prune_preserves_full_latest_messages(saver):
+    def echo(state):
+        last = state["messages"][-1].content
+        return {"messages": [AIMessage(content=f"reply {last}")]}
+
+    graph = StateGraph(MessagesState)
+    graph.add_node("answer", echo)
+    graph.add_edge(START, "answer")
+    graph.add_edge("answer", END)
+    agent = graph.compile(checkpointer=saver)
+
+    thread_id = "many-turns"
+    config = {"configurable": {"thread_id": thread_id}}
+    turn_count = 25
+    for i in range(turn_count):
+        await agent.ainvoke(
+            {"messages": [HumanMessage(content=f"msg {i}")]}, config=config
+        )
+
+    assert len(await _checkpoint_ids(saver, thread_id)) > DEFAULT_KEEP_PER_THREAD
+
+    await prune_thread_checkpoints(saver, thread_id, keep=5)
+
+    assert len(await _checkpoint_ids(saver, thread_id)) == 5
+    state = await agent.aget_state(config)
+    # Every turn's human + AI message is still present: pruning dropped old
+    # checkpoint rows, but the newest row's channel_values already holds the
+    # full accumulated message list (not a delta).
+    assert len(state.values["messages"]) == turn_count * 2
+    assert state.values["messages"][0].content == "msg 0"
+    assert state.values["messages"][-1].content == f"reply msg {turn_count - 1}"
+
+
+@pytest.fixture
+async def conv_app(tmp_path):
+    await Tortoise.init(
+        db_url=f"sqlite://{tmp_path / 'conv.sqlite'}",
+        modules={
+            "models": ["blueprints.users.models", "blueprints.conversation.models"]
+        },
+    )
+    await Tortoise.generate_schemas()
+    app = Quart(__name__)
+    app.config.update(
+        TESTING=True,
+        JWT_SECRET_KEY="checkpoint-test-secret-at-least-thirty-two-characters",
+    )
+    JWTManager(app)
+    app.register_blueprint(conversation_blueprint)
+    user = await User.create(username="owner", email="owner@test")
+    async with app.app_context():
+        headers = {
+            "Authorization": f"Bearer {create_access_token(identity=str(user.id))}"
+        }
+    try:
+        yield app, headers, user
+    finally:
+        await Tortoise.close_connections()
+
+
+async def test_delete_conversation_via_api_removes_its_thread_and_leaves_others(
+    conv_app, tmp_path
+):
+    app, headers, user = conv_app
+    conv1 = await Conversation.create(user=user, name="one")
+    conv2 = await Conversation.create(user=user, name="two")
+
+    async with aiosqlite.connect(tmp_path / "checkpoints.db") as conn:
+        s = AsyncSqliteSaver(conn)
+        await s.setup()
+        await _add_checkpoint(s, str(conv1.id))
+        await _add_checkpoint(s, str(conv2.id))
+        app.main_agent = SimpleNamespace(checkpointer=s)
+
+        response = await app.test_client().delete(
+            f"/api/conversation/{conv1.id}", headers=headers
+        )
+        assert response.status_code == 200
+        assert (await response.get_json())["success"] is True
+        assert await Conversation.get_or_none(id=conv1.id) is None
+
+        assert await _checkpoint_ids(s, str(conv1.id)) == []
+        assert len(await _checkpoint_ids(s, str(conv2.id))) == 1
+
+
+async def test_delete_conversation_succeeds_even_when_checkpoint_delete_fails(
+    conv_app,
+):
+    app, headers, user = conv_app
+    conv = await Conversation.create(user=user, name="doomed")
+    app.main_agent = SimpleNamespace(
+        checkpointer=SimpleNamespace(
+            adelete_thread=AsyncMock(side_effect=RuntimeError("sqlite is gone"))
+        )
+    )
+
+    response = await app.test_client().delete(
+        f"/api/conversation/{conv.id}", headers=headers
+    )
+    assert response.status_code == 200
+    assert (await response.get_json())["success"] is True
+    assert await Conversation.get_or_none(id=conv.id) is None
+
+
+async def test_delete_thread_checkpoints_swallows_errors(caplog):
+    checkpointer = SimpleNamespace(
+        adelete_thread=AsyncMock(side_effect=RuntimeError("boom"))
+    )
+    await delete_thread_checkpoints(checkpointer, "t")
+    assert "Failed to delete checkpoint thread" in caplog.text
+
+
+@pytest.mark.parametrize("raw", ["0", "-5", "not-a-number"])
+def test_keep_per_thread_rejects_invalid_values(monkeypatch, caplog, raw):
+    monkeypatch.setenv("CHECKPOINT_KEEP_PER_THREAD", raw)
+    assert keep_per_thread() == DEFAULT_KEEP_PER_THREAD
+    assert "invalid" in caplog.text
+
+
+def test_keep_per_thread_accepts_valid_override(monkeypatch):
+    monkeypatch.setenv("CHECKPOINT_KEEP_PER_THREAD", "7")
+    assert keep_per_thread() == 7

tests/unit/test_configuration.py

@@ -0,0 +1,319 @@
+"""Startup policy and the environment contract shared by blue/green services."""
+
+import base64
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from config.validation import ConfigurationError, validate_configuration
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+@pytest.fixture
+def production_env():
+    return {
+        "DATABASE_URL": "postgres://raggr:a-random-test-password@postgres:5432/raggr",
+        "JWT_SECRET_KEY": "a-long-test-signing-secret-with-32-characters",
+        "FERNET_KEY": base64.urlsafe_b64encode(bytes(range(32))).decode(),
+        "OPENAI_API_KEY": "synthetic-test-api-key",
+    }
+
+
+def test_optional_integrations_are_not_required(production_env):
+    validate_configuration(production_env)
+    validate_configuration({}, production=False)
+
+
+@pytest.mark.parametrize(
+    "name", ["DATABASE_URL", "JWT_SECRET_KEY", "FERNET_KEY", "OPENAI_API_KEY"]
+)
+def test_missing_required_settings(production_env, name):
+    del production_env[name]
+    with pytest.raises(ConfigurationError, match=name):
+        validate_configuration(production_env)
+
+
+@pytest.mark.parametrize(
+    "name,value",
+    [
+        ("JWT_SECRET_KEY", "SECRET_KEY"),
+        ("JWT_SECRET_KEY", "your-" + "x" * 40),
+        ("JWT_SECRET_KEY", "short"),
+        ("OPENAI_API_KEY", "your-openai-api-key"),
+        ("FERNET_KEY", "not-base64"),
+        ("FERNET_KEY", base64.urlsafe_b64encode(b"short").decode()),
+        ("FERNET_KEY", base64.b64encode(b"\xff" * 32).decode()),
+        ("DATABASE_URL", "sqlite://test.db"),
+        ("DATABASE_URL", "postgres://raggr:change%6de@postgres/raggr"),
+        ("DATABASE_URL", "postgres://raggr:password@postgres/raggr"),
+        ("DATABASE_URL", "postgres://raggr:strong@postgres:70000/raggr"),
+        ("DATABASE_URL", "postgres://raggr@postgres/raggr"),
+        ("LLAMA_SERVER_URL", "localhost:8080/v1"),
+        ("EMBEDDING_SERVER_URL", "http://server:invalid/v1"),
+        ("LLAMA_SERVER_URL", "http://user:private@server/v1"),
+        ("LLAMA_MODEL_NAME", ""),
+        ("METRICS_TOKEN", "short"),
+        ("CHECKPOINT_DB_PATH", ":memory:"),
+        ("CHECKPOINT_DB_PATH", "/data/invalid\x00.db"),
+        ("CONVERSATION_SUMMARY_DB_PATH", ""),
+        ("OBSIDIAN_CONTINUOUS_SYNC", "yes"),
+        ("OBSIDIAN_SYNC_MODE", "push-only"),
+    ],
+)
+def test_malformed_and_unsafe_settings(production_env, name, value):
+    production_env[name] = value
+    with pytest.raises(ConfigurationError, match=name) as error:
+        validate_configuration(production_env)
+    if value and value not in name:
+        assert value not in str(error.value)
+
+
+@pytest.mark.parametrize("value", ["0", "-1", "abc", "", "nan", "inf", "-inf"])
+@pytest.mark.parametrize(
+    "name",
+    [
+        "TTFT_TIMEOUT_SECONDS",
+        "LLAMA_TIMEOUT_SECONDS",
+        "PROMPT_BUDGET_TOKENS",
+        "WHATSAPP_RATE_LIMIT_MAX",
+    ],
+)
+def test_invalid_numbers(production_env, name, value):
+    production_env[name] = value
+    with pytest.raises(ConfigurationError, match=name):
+        validate_configuration(production_env)
+
+
+def test_context_budget_boundary(production_env):
+    production_env.update(
+        CONTEXT_WINDOW_TOKENS="100",
+        PROMPT_BUDGET_TOKENS="80",
+        RESPONSE_RESERVE_TOKENS="20",
+    )
+    validate_configuration(production_env)
+    production_env["RESPONSE_RESERVE_TOKENS"] = "21"
+    with pytest.raises(ConfigurationError, match="exceeds context window"):
+        validate_configuration(production_env)
+
+
+def test_oidc_partial_insecure_and_manual_endpoints(production_env):
+    production_env["OIDC_ISSUER"] = "https://auth.example.test"
+    with pytest.raises(ConfigurationError, match="OIDC_CLIENT_ID"):
+        validate_configuration(production_env)
+    production_env.update(
+        OIDC_CLIENT_ID="raggr",
+        OIDC_CLIENT_SECRET="synthetic-client-secret",
+        OIDC_REDIRECT_URI="http://localhost:8080/api/user/oidc/callback",
+    )
+    with pytest.raises(ConfigurationError, match="OIDC_REDIRECT_URI"):
+        validate_configuration(production_env)
+    production_env["OIDC_REDIRECT_URI"] = (
+        "https://app.example.test/api/user/oidc/callback"
+    )
+    validate_configuration(production_env)
+    production_env["OIDC_USE_DISCOVERY"] = "false"
+    with pytest.raises(ConfigurationError, match="OIDC_TOKEN_ENDPOINT"):
+        validate_configuration(production_env)
+    for name in (
+        "OIDC_AUTHORIZATION_ENDPOINT",
+        "OIDC_TOKEN_ENDPOINT",
+        "OIDC_USERINFO_ENDPOINT",
+        "OIDC_JWKS_URI",
+    ):
+        production_env[name] = "https://auth.example.test/endpoint"
+    validate_configuration(production_env)
+
+
+@pytest.mark.parametrize(
+    "trigger,key,flag",
+    [
+        ("TWILIO_ACCOUNT_SID", "TWILIO_AUTH_TOKEN", "TWILIO_SIGNATURE_VALIDATION"),
+        (
+            "MAILGUN_DOMAIN",
+            "MAILGUN_WEBHOOK_SIGNING_KEY",
+            "MAILGUN_SIGNATURE_VALIDATION",
+        ),
+        (
+            "SENDBLUE_API_KEY",
+            "SENDBLUE_WEBHOOK_SECRET",
+            "SENDBLUE_SIGNATURE_VALIDATION",
+        ),
+    ],
+)
+def test_enabled_webhooks_require_signatures(production_env, trigger, key, flag):
+    production_env[trigger] = "synthetic-test-value"
+    with pytest.raises(ConfigurationError, match=key):
+        validate_configuration(production_env)
+    production_env[key] = "synthetic-signing-key"
+    validate_configuration(production_env)
+    production_env[flag] = "false"
+    with pytest.raises(ConfigurationError, match=flag):
+        validate_configuration(production_env)
+
+
+@pytest.mark.parametrize(
+    "flag",
+    [
+        "TWILIO_SIGNATURE_VALIDATION",
+        "MAILGUN_SIGNATURE_VALIDATION",
+        "SENDBLUE_SIGNATURE_VALIDATION",
+    ],
+)
+def test_production_never_disables_webhook_verification(production_env, flag):
+    production_env[flag] = "false"
+    with pytest.raises(ConfigurationError, match=flag):
+        validate_configuration(production_env)
+    validate_configuration(production_env, production=False)
+
+
+def test_sync_and_summary_requirements(production_env):
+    production_env["OBSIDIAN_CONTINUOUS_SYNC"] = "true"
+    with pytest.raises(ConfigurationError, match="OBSIDIAN_VAULT_ID"):
+        validate_configuration(production_env)
+    production_env.update(
+        OBSIDIAN_EMAIL="test@example.test",
+        OBSIDIAN_PASSWORD="test-password",
+        OBSIDIAN_VAULT_ID="test-vault",
+    )
+    validate_configuration(production_env)
+    production_env.update(
+        CHECKPOINT_DB_PATH="/data/store.db",
+        CONVERSATION_SUMMARY_DB_PATH="/data/../data/store.db",
+    )
+    with pytest.raises(ConfigurationError, match="must differ"):
+        validate_configuration(production_env)
+
+
+def test_compose_forwards_runtime_settings():
+    text = (ROOT / "docker-compose.yml").read_text()
+    common = text.split("services:")[0]
+    for name in (
+        "METRICS_TOKEN",
+        "TTFT_TIMEOUT_SECONDS",
+        "LLAMA_TIMEOUT_SECONDS",
+        "CONTEXT_WINDOW_TOKENS",
+        "PROMPT_BUDGET_TOKENS",
+        "RESPONSE_RESERVE_TOKENS",
+        "CHECKPOINT_DB_PATH",
+        "CONVERSATION_SUMMARY_DB_PATH",
+        "OBSIDIAN_SYNC_INTERVAL",
+        "OBSIDIAN_SYNC_MODE",
+        "OIDC_AUTHORIZATION_ENDPOINT",
+        "OIDC_TOKEN_ENDPOINT",
+        "OIDC_USERINFO_ENDPOINT",
+        "OIDC_JWKS_URI",
+        "EMAIL_HMAC_SECRET",
+        "WHATSAPP_RATE_LIMIT_MAX",
+        "WHATSAPP_RATE_LIMIT_WINDOW",
+        "IMESSAGE_RATE_LIMIT_MAX",
+        "IMESSAGE_RATE_LIMIT_WINDOW",
+    ):
+        assert f"- {name}=${{{name}" in common
+    assert "TAVILY_API_KEY=${TAVILY_API_KEY:-${TAVILIY_API_KEY:-}}" in common
+    assert text.count("<<: *raggr-common") == 2
+
+
+@pytest.mark.parametrize(
+    "canonical,legacy,expected",
+    [("new-key", "old-key", "new-key"), ("", "old-key", "old-key")],
+)
+def test_compose_interpolation_without_secrets(canonical, legacy, expected):
+    if not shutil.which("docker"):
+        pytest.skip("Docker Compose is unavailable")
+    env = {
+        "PATH": os.environ["PATH"],
+        "HOME": os.environ.get("HOME", "/tmp"),
+        "TAVILY_API_KEY": canonical,
+        "TAVILIY_API_KEY": legacy,
+        "METRICS_TOKEN": "test-metrics",
+        "TTFT_TIMEOUT_SECONDS": "42",
+        "CHECKPOINT_DB_PATH": "/data/custom.db",
+    }
+    result = subprocess.run(
+        [
+            "docker",
+            "compose",
+            "--env-file",
+            "/dev/null",
+            "--profile",
+            "blue",
+            "--profile",
+            "green",
+            "config",
+            "--format",
+            "json",
+        ],
+        cwd=ROOT,
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode and "not a docker command" in result.stderr:
+        pytest.skip("Docker Compose is unavailable")
+    assert result.returncode == 0, result.stderr
+    services = json.loads(result.stdout)["services"]
+    for color in ("blue", "green"):
+        actual = services[f"raggr-{color}"]["environment"]
+        assert actual["TAVILY_API_KEY"] == expected
+        assert actual["METRICS_TOKEN"] == "test-metrics"
+        assert actual["TTFT_TIMEOUT_SECONDS"] == "42"
+        assert actual["CONVERSATION_SUMMARY_DB_PATH"] == "/data/custom.db.summaries.db"
+
+
+@pytest.mark.parametrize(
+    "validation_status,migration_status,expected", [(0, 19, 19), (23, 0, 23)]
+)
+def test_startup_stops_before_app_on_failure(
+    tmp_path, validation_status, migration_status, expected
+):
+    log = tmp_path / "commands"
+    for name, body in {
+        "python": f'echo "python $*" >> "$COMMAND_LOG"\nexit {validation_status}\n',
+        "aerich": f'echo "aerich $*" >> "$COMMAND_LOG"\nexit {migration_status}\n',
+    }.items():
+        executable = tmp_path / name
+        executable.write_text("#!/bin/sh\n" + body)
+        executable.chmod(0o755)
+    result = subprocess.run(
+        ["/bin/bash", str(ROOT / "startup.sh")],
+        cwd=ROOT,
+        env={"PATH": str(tmp_path), "COMMAND_LOG": str(log)},
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == expected
+    commands = log.read_text()
+    assert "python scripts/validate_config.py" in commands
+    assert "python app.py" not in commands
+    assert ("aerich upgrade" in commands) == (validation_status == 0)
+
+
+def test_validation_cli_reports_errors_without_values(tmp_path, production_env):
+    # A separate project tree avoids loading any real .env in the shared checkout.
+    (tmp_path / "scripts").mkdir()
+    (tmp_path / "config").mkdir()
+    (tmp_path / "config" / "__init__.py").touch()
+    shutil.copy(ROOT / "scripts" / "validate_config.py", tmp_path / "scripts")
+    shutil.copy(ROOT / "config" / "validation.py", tmp_path / "config")
+    command = [sys.executable, str(tmp_path / "scripts" / "validate_config.py")]
+    env = {"PATH": os.environ["PATH"], **production_env}
+    result = subprocess.run(command, env=env, capture_output=True, text=True)
+    assert result.returncode == 0
+    assert result.stdout == "Configuration valid.\n"
+    env["JWT_SECRET_KEY"] = "private-invalid-value"
+    result = subprocess.run(command, env=env, capture_output=True, text=True)
+    assert result.returncode == 1
+    assert "JWT_SECRET_KEY" in result.stderr
+    assert env["JWT_SECRET_KEY"] not in result.stderr
+    assert env["OPENAI_API_KEY"] not in result.stderr
+    # dotenv values are validated too, but do not override exported settings.
+    (tmp_path / ".env").write_text("JWT_SECRET_KEY=invalid-dotenv-value\n")
+    env["JWT_SECRET_KEY"] = production_env["JWT_SECRET_KEY"]
+    result = subprocess.run(command, env=env, capture_output=True, text=True)
+    assert result.returncode == 0

tests/unit/test_context_summary.py

@@ -102,7 +102,8 @@ async def test_tools_and_incomplete_exchanges_do_not_become_summary_instructions
     assert "Now continue" not in message.content
     prompt = build_context(messages, "trusted system", [], summary_message=message)
     assert prompt[0].content == "trusted system"
-    assert prompt[1] == message
+    # Nothing was dropped, so the ledger would only duplicate retained turns.
+    assert message not in prompt
     assert prompt[-1] is messages[-1]
 
 
@@ -162,3 +163,17 @@ async def test_deletion_is_scoped_and_survives_reopen(tmp_path):
     await store.delete("owner", "c")
     assert (await SummaryStore(path).load("owner", "c")).cursor == 0
     assert (await SummaryStore(path).load("other", "c")).cursor == 4
+
+
+async def test_largest_ledger_fits_the_fixed_summary_reserve(tmp_path):
+    from blueprints.conversation.context import SUMMARY_RESERVE_TOKENS, message_tokens
+
+    messages = []
+    for i in range(100):
+        messages += [
+            HumanMessage(content=f"Must {i}: " + "猫😺" * 1000),
+            AIMessage(content=f"Decided {i}: " + "budget " * 1000),
+        ]
+    messages.append(HumanMessage(content="current"))
+    state = await SummaryStore(tmp_path / "summary.db").update("u", "c", messages)
+    assert message_tokens(state.as_message()) <= SUMMARY_RESERVE_TOKENS

tests/unit/test_context_trim.py

@@ -462,3 +462,156 @@ def test_focused_note_heading_index_uses_absolute_content_lines():
     metadata = json.loads(projected.content)["data"]["markdown_compaction"]
     assert metadata["headings"] == [{"line": 1001, "heading": "# Requested section"}]
     assert msg.artifact["data"]["markdown"] == "full original"
+
+
+def _search_artifact(chunk_text: str) -> dict:
+    return {
+        "kind": "search.documents",
+        "title": "Search: vet",
+        "data": {
+            "query": "vet",
+            "source": "paperless",
+            "results": [
+                {
+                    "snippet": chunk_text[:50],
+                    "title": "vet_invoice",
+                    "source_id": "chunk-1",
+                    "url": "/sources/chunk-1",
+                }
+            ],
+        },
+        "model_context": {
+            "results": [
+                {
+                    "content": chunk_text,
+                    "title": "vet_invoice",
+                    "source_id": "chunk-1",
+                    "url": "/sources/chunk-1",
+                }
+            ]
+        },
+    }
+
+
+def test_search_result_compaction_keeps_full_chunk_text_under_a_generous_limit():
+    import json
+
+    chunk_text = "Rabies vaccine administered. " + "detail " * 200
+    msg = ToolMessage(
+        content="display", artifact=_search_artifact(chunk_text), tool_call_id="a"
+    )
+    projected = context.compact_tool_result(msg, 100_000)
+    parsed = json.loads(projected.content)
+    (result,) = parsed["data"]["results"]
+    assert result["content"] == chunk_text
+    assert "snippet" not in result
+
+
+def test_search_result_compaction_shortens_content_but_keeps_source_ids_under_pressure():
+    import json
+
+    chunk_text = "Rabies vaccine administered. " + "detail " * 200
+    msg = ToolMessage(
+        content="display", artifact=_search_artifact(chunk_text), tool_call_id="a"
+    )
+    projected = context.compact_tool_result(msg, 700)
+    parsed = json.loads(projected.content)
+    (result,) = parsed["data"]["results"]
+    assert len(result["content"]) < len(chunk_text)
+    assert result["source_id"] == "chunk-1"
+    assert result["url"] == "/sources/chunk-1"
+
+
+def _conversation(turns, size=1000):
+    messages = []
+    for i in range(turns):
+        messages += [
+            HumanMessage(content=f"Question {i}"),
+            call(str(i)),
+            result(str(i), "data " * size),
+            AIMessage(content=f"Decision {i}"),
+        ]
+    return messages
+
+
+def test_summary_omitted_when_no_turns_dropped():
+    messages = _conversation(2, size=10) + [HumanMessage(content="next")]
+    summary = AIMessage(content="Earlier: keep dining low")
+    assert trim_history(messages, summary_message=summary) == trim_history(messages)
+    assert summary not in trim_history(messages, summary_message=summary)
+
+
+def assert_valid_sequence(out):
+    seen = set()
+    for m in out:
+        if isinstance(m, AIMessage):
+            seen.update(c["id"] for c in m.tool_calls)
+        if isinstance(m, ToolMessage):
+            assert m.tool_call_id in seen
+
+
+def test_summary_placed_before_current_turn_when_turns_dropped():
+    question = HumanMessage(content="next")
+    # The current turn has its own tool exchange the summary must not split.
+    messages = _conversation(12) + [question, call("now"), result("now")]
+    summary = AIMessage(content="Earlier: keep dining low")
+    out = trim_history(messages, token_budget=6000, summary_message=summary)
+    assert cost(out) <= 6000
+    assert out[-4:] == [summary, question, messages[-2], messages[-1]]
+    assert out.count(summary) == 1
+    # The ledger supersedes excerpts of dropped turns; don't send both.
+    assert not any(
+        isinstance(m.content, str)
+        and m.content.startswith("Earlier conversation excerpts")
+        for m in out
+    )
+    assert isinstance(out[0], HumanMessage)
+    # Without a ledger, the excerpts still cover the dropped turns.
+    plain = trim_history(messages, token_budget=6000)
+    assert plain[0].content.startswith("Earlier conversation excerpts")
+    assert_valid_sequence(out)
+
+
+def test_changing_summary_leaves_retained_prefix_identical():
+    messages = _conversation(12) + [HumanMessage(content="next")]
+    short = AIMessage(content="Earlier: keep dining low")
+    long = AIMessage(content="Earlier: " + "constraint " * 400)
+    a = trim_history(messages, token_budget=6000, summary_message=short)
+    b = trim_history(messages, token_budget=6000, summary_message=long)
+    assert a[: a.index(short)] == b[: b.index(long)]
+
+
+def test_trim_drops_to_low_watermark_and_prefix_stays_stable():
+    budget = 8000
+    messages, oldest, costs = [], [], []
+    for i in range(30):
+        messages.append(HumanMessage(content=f"Question {i}"))
+        out = trim_history(messages, token_budget=budget)
+        assert out[-1] is messages[-1]
+        assert cost(out) <= budget
+        oldest.append(next(m for m in out if isinstance(m, HumanMessage)).content)
+        costs.append(cost(out))
+        messages += [call(str(i)), result(str(i), "data " * 300), AIMessage("ok")]
+    cuts = [i for i in range(1, 30) if oldest[i] != oldest[i - 1]]
+    assert cuts, "history should overflow the budget"
+    # Each drop goes well below the budget instead of one turn under it...
+    for i in cuts:
+        assert costs[i] <= budget * (context.TRIM_LOW_WATERMARK + 0.1)
+    # ...so the retained prefix is unchanged for several turns in between.
+    assert all(b - a >= 3 for a, b in zip(cuts, cuts[1:]))
+    assert len(cuts) <= 5
+
+
+def test_search_result_compaction_keeps_source_ids_when_no_rows_fit():
+    import json
+
+    chunk_text = "Rabies vaccine administered. " + "detail " * 200
+    msg = ToolMessage(
+        content="display", artifact=_search_artifact(chunk_text), tool_call_id="a"
+    )
+    projected = json.loads(context.compact_tool_result(msg, 300).content)
+    data = projected["data"]
+    assert data["results"] == []
+    assert data["results_compaction"]["omitted_source_refs"] == [
+        {"source_id": "chunk-1", "url": "/sources/chunk-1"}
+    ]

tests/unit/test_file_upload.py

@@ -1,6 +1,7 @@
 """Tests for conversation file attachments: extraction, /upload and stream-query file_key."""
 
 import io
+import threading
 from datetime import timedelta
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, patch
@@ -190,6 +191,21 @@ class TestUploadRoute:
         s3.upload_image.assert_awaited_once()
         s3.upload_file.assert_not_awaited()
 
+    async def test_image_processing_runs_off_the_event_loop(
+        self, app, headers, s3, monkeypatch
+    ):
+        threads = []
+        real = conv.process_image
+
+        def recording(*args):
+            threads.append(threading.current_thread())
+            return real(*args)
+
+        monkeypatch.setattr(conv, "process_image", recording)
+        resp = await _upload(app, headers, _png_bytes(), "cat.png", "image/png")
+        assert resp.status_code == 200
+        assert threads and threads[0] is not threading.main_thread()
+
     async def test_unsupported_type_is_415(self, app, headers, s3):
         resp = await _upload(app, headers, b"PK\x03\x04", "a.zip", "application/zip")
         assert resp.status_code == 415
@@ -247,10 +263,14 @@ class TestUploadRoute:
 
 class TestStreamQueryFileKey:
     async def test_rejects_file_key_from_other_conversation(self, app, headers, s3):
+        from blueprints.conversation import runs
+
         add = AsyncMock()
+        owned = AsyncMock(return_value=SimpleNamespace(id=CONV_ID))
         with (
-            patch.object(conv.blueprints.users.models.User, "get", AsyncMock()),
-            patch.object(conv, "add_message_to_conversation", add),
+            patch.object(runs.Conversation, "get_or_none", owned),
+            patch.object(runs.ChatRun, "create", AsyncMock()) as create,
+            patch.object(runs, "add_message_to_conversation", add),
         ):
             resp = await app.test_client().post(
                 "/api/conversation/stream-query",
@@ -262,6 +282,7 @@ class TestStreamQueryFileKey:
                 },
             )
         assert resp.status_code == 400
+        create.assert_not_awaited()
         add.assert_not_awaited()
 
     async def test_load_attached_file_recovers_name_and_text(self):
@@ -280,3 +301,45 @@ class TestStreamQueryFileKey:
                 f"conversations/{CONV_ID}/files/u.txt"
             )
         assert name == "u.txt"
+
+
+class TestImageOwnership:
+    async def test_presigns_only_keys_in_owned_conversations(self, app, headers, s3):
+        presign = AsyncMock(return_value="https://s3.example/signed")
+        with patch.object(conv, "s3_presigned_url", presign):
+            client = app.test_client()
+            own = await client.get(
+                f"/api/conversation/image/conversations/{CONV_ID}/a.jpg",
+                headers=headers,
+            )
+            assert own.status_code == 200
+            for key in (
+                f"conversations/{OTHER_ID}/a.jpg",
+                f"conversations/{CONV_ID}/../{OTHER_ID}/a.jpg",
+                "conversations/not-a-uuid/a.jpg",
+                "beli/profile.jpg",
+            ):
+                resp = await client.get(
+                    f"/api/conversation/image/{key}", headers=headers
+                )
+                assert resp.status_code == 404, key
+        presign.assert_awaited_once_with(f"conversations/{CONV_ID}/a.jpg")
+
+    async def test_image_upload_requires_owned_conversation(self, app, headers, s3):
+        with patch.object(conv.blueprints.users.models.User, "get", AsyncMock()):
+            resp = await self._upload_image(app, headers)
+        assert resp.status_code == 404
+        s3.upload_image.assert_not_awaited()
+
+    @staticmethod
+    async def _upload_image(app, headers):
+        return await app.test_client().post(
+            "/api/conversation/upload-image",
+            headers=headers,
+            form={"conversation_id": OTHER_ID},
+            files={
+                "file": FileStorage(
+                    io.BytesIO(_png_bytes()), filename="a.png", content_type="image/png"
+                )
+            },
+        )

tests/unit/test_history_management.py

@@ -0,0 +1,260 @@
+"""Real SQLite tests: ownership, joined pagination, exports and malformed requests."""
+
+import base64
+import importlib.util
+import json
+from pathlib import Path
+import zlib
+
+import pytest
+from quart import Quart
+from quart_jwt_extended import JWTManager, create_access_token
+from tortoise import Tortoise
+
+from blueprints.conversation.history import history_blueprint, list_owned_conversations
+from blueprints.conversation.models import (
+    Conversation,
+    ConversationMessage,
+    Speaker,
+    UserMemory,
+)
+from blueprints.users.models import User
+
+
+@pytest.fixture
+async def history_app():
+    await Tortoise.init(
+        db_url="sqlite://:memory:",
+        modules={
+            "models": ["blueprints.users.models", "blueprints.conversation.models"]
+        },
+    )
+    await Tortoise.generate_schemas()
+    app = Quart(__name__)
+    app.config.update(
+        TESTING=True,
+        JWT_SECRET_KEY="history-test-secret-at-least-thirty-two-characters",
+    )
+    JWTManager(app)
+    app.register_blueprint(history_blueprint)
+    owner = await User.create(username="owner", email="owner@example.com")
+    foreign = await User.create(username="foreign", email="foreign@example.com")
+    async with app.app_context():
+        headers = {
+            "Authorization": f"Bearer {create_access_token(identity=str(owner.id))}"
+        }
+    try:
+        yield app.test_client(), headers, owner, foreign
+    finally:
+        await Tortoise.close_connections()
+
+
+async def message(conversation, text, speaker=Speaker.USER, **kwargs):
+    return await ConversationMessage.create(
+        conversation=conversation, text=text, speaker=speaker, **kwargs
+    )
+
+
+@pytest.mark.asyncio
+async def test_search_unique_pagination_and_owner_isolation(history_app):
+    client, headers, owner, foreign = history_app
+    a = await Conversation.create(user=owner, name="ordinary")
+    b = await Conversation.create(user=owner, name="needle title", pinned=True)
+    secret = await Conversation.create(user=foreign, name="needle foreign", pinned=True)
+    await Conversation.create(name="needle unowned")
+    for _ in range(5):
+        await message(a, "many needle message matches")
+    await message(secret, "needle secret")
+    first = await list_owned_conversations(owner.id, "needle", 1, 0)
+    second = await list_owned_conversations(owner.id, "needle", 1, 1)
+    assert [first[0]["id"], second[0]["id"]] == [str(b.id), str(a.id)]
+    assert await list_owned_conversations(owner.id, "needle", 1, 2) == []
+    assert "needle" in second[0]["snippet"]
+    assert second[0]["match_message_id"]
+    response = await client.get(
+        "/api/history/conversations?search=NEEDLE&limit=10", headers=headers
+    )
+    assert {row["id"] for row in await response.get_json()} == {str(a.id), str(b.id)}
+
+
+@pytest.mark.asyncio
+async def test_pin_owner_only_boolean_and_recency(history_app):
+    client, headers, owner, foreign = history_app
+    own = await Conversation.create(user=owner, name="own")
+    other = await Conversation.create(user=foreign, name="other")
+    for identifier in (other.id, "malformed"):
+        response = await client.patch(
+            f"/api/history/conversations/{identifier}/pin",
+            headers=headers,
+            json={"pinned": True},
+        )
+        assert response.status_code == 404
+    for body in ({}, {"pinned": "false"}, {"pinned": 1}, [], None):
+        response = await client.patch(
+            f"/api/history/conversations/{own.id}/pin", headers=headers, json=body
+        )
+        assert response.status_code == 400
+    original = own.updated_at
+    for pinned in (True, False):
+        response = await client.patch(
+            f"/api/history/conversations/{own.id}/pin",
+            headers=headers,
+            json={"pinned": pinned},
+        )
+        assert response.status_code == 200
+        await own.refresh_from_db()
+        assert own.pinned is pinned
+        assert own.updated_at == original
+    await other.refresh_from_db()
+    assert not other.pinned
+
+
+@pytest.mark.asyncio
+async def test_export_display_and_attachment_provenance(history_app):
+    client, headers, owner, foreign = history_app
+    own = await Conversation.create(user=owner, name='quoted "title"')
+    other = await Conversation.create(user=foreign, name="FOREIGN SECRET")
+    await message(other, "FOREIGN CONTENT")
+    await message(own, "hello\n\n📎 notes.md", image_key="foreign/private/secret.png")
+    await message(own, "[web_search result]\nHIDDEN RAW TOOL CONTEXT", Speaker.TOOL)
+    await message(own, "answer", Speaker.SIMBA)
+    await message(own, "malformed tool payload\nHIDDEN MORE", Speaker.TOOL)
+    response = await client.get(
+        f"/api/history/conversations/{own.id}/export", headers=headers
+    )
+    assert response.status_code == 200
+    data = await response.get_json()
+    text = json.dumps(data)
+    for secret in ("HIDDEN RAW", "HIDDEN MORE", "FOREIGN", "foreign/private"):
+        assert secret not in text
+    assert data["messages"][1]["text"] == "[web_search result]"
+    assert data["messages"][1]["provenance"] == "displayed_tool_header"
+    assert data["messages"][0]["attachments"] == [
+        {"kind": "file", "recorded_name": "notes.md", "contents_included": False},
+        {"kind": "image", "stored_reference_present": True, "contents_included": False},
+    ]
+    assert "attachment; filename=" in response.headers["Content-Disposition"]
+    assert response.headers["Cache-Control"] == "no-store"
+    md = await client.get(
+        f"/api/history/conversations/{own.id}/export?format=markdown", headers=headers
+    )
+    markdown = await md.get_data(as_text=True)
+    assert md.status_code == 200 and "text/markdown" in md.content_type
+    assert "📎 notes.md" in markdown and "Raw tool context" in markdown
+    assert "HIDDEN" not in markdown
+    for identifier in (other.id, "bad-uuid"):
+        assert (
+            await client.get(
+                f"/api/history/conversations/{identifier}/export", headers=headers
+            )
+        ).status_code == 404
+    assert (
+        await client.get(
+            f"/api/history/conversations/{own.id}/export?format=html", headers=headers
+        )
+    ).status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_memory_owner_only_correction_forget_and_validation(history_app):
+    client, headers, owner, foreign = history_app
+    own = await UserMemory.create(user=owner, content="old fact")
+    other = await UserMemory.create(user=foreign, content="FOREIGN MEMORY")
+    rows = await (await client.get("/api/history/memories", headers=headers)).get_json()
+    assert [row["id"] for row in rows] == [str(own.id)]
+    for identifier in (other.id, "invalid"):
+        for method in (client.patch, client.delete):
+            assert (
+                await method(
+                    f"/api/history/memories/{identifier}",
+                    headers=headers,
+                    json={"content": "stolen"},
+                )
+            ).status_code == 404
+    for body in (
+        {},
+        [],
+        {"content": None},
+        {"content": 3},
+        {"content": "  "},
+        {"content": "x" * 10001},
+    ):
+        assert (
+            await client.patch(
+                f"/api/history/memories/{own.id}", headers=headers, json=body
+            )
+        ).status_code == 400
+    response = await client.patch(
+        f"/api/history/memories/{own.id}",
+        headers=headers,
+        json={"content": " corrected fact "},
+    )
+    assert response.status_code == 200
+    assert (await response.get_json())["content"] == "corrected fact"
+    await own.refresh_from_db()
+    assert own.content == "corrected fact"
+    assert (
+        await client.delete(f"/api/history/memories/{own.id}", headers=headers)
+    ).status_code == 204
+    assert await UserMemory.get_or_none(id=own.id) is None
+    await other.refresh_from_db()
+    assert other.content == "FOREIGN MEMORY"
+
+
+@pytest.mark.asyncio
+async def test_memory_pagination(history_app):
+    client, headers, owner, _ = history_app
+    for i in range(5):
+        await UserMemory.create(user=owner, content=f"fact {i}")
+    first = await (
+        await client.get("/api/history/memories?limit=2&offset=0", headers=headers)
+    ).get_json()
+    second = await (
+        await client.get("/api/history/memories?limit=2&offset=2", headers=headers)
+    ).get_json()
+    assert len(first) == len(second) == 2
+    assert {r["id"] for r in first}.isdisjoint(r["id"] for r in second)
+
+
+@pytest.mark.asyncio
+async def test_authentication_and_malformed_pagination(history_app):
+    client, headers, owner, _ = history_app
+    own = await Conversation.create(user=owner, name="own")
+    memory = await UserMemory.create(user=owner, content="fact")
+    for method, path in (
+        (client.get, "/conversations"),
+        (client.get, "/memories"),
+        (client.get, f"/conversations/{own.id}/export"),
+        (client.patch, f"/conversations/{own.id}/pin"),
+        (client.patch, f"/memories/{memory.id}"),
+        (client.delete, f"/memories/{memory.id}"),
+    ):
+        assert (await method("/api/history" + path)).status_code == 401
+    for path in ("conversations", "memories"):
+        for query in ("limit=0", "limit=101", "offset=-1", "limit=oops", "offset=1.5"):
+            assert (
+                await client.get(f"/api/history/{path}?{query}", headers=headers)
+            ).status_code == 400
+    assert (
+        await client.get(
+            "/api/history/conversations?search=" + "x" * 501, headers=headers
+        )
+    ).status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_migration_adds_pinned_and_records_state(history_app):
+    migration_path = (
+        Path(__file__).parents[2]
+        / "migrations/models/20_20261012231017_application_foundation.py"
+    )
+    spec = importlib.util.spec_from_file_location("history_migration", migration_path)
+    migration = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(migration)
+    sql = await migration.upgrade(None)
+    assert 'ALTER TABLE "conversations" ADD "pinned" BOOL NOT NULL DEFAULT False' in sql
+    state = json.loads(zlib.decompress(base64.b64decode(migration.MODELS_STATE)))
+    assert any(
+        field["name"] == "pinned" and field["default"] is False
+        for field in state["models.Conversation"]["data_fields"]
+    )

tests/unit/test_integration_status.py

@@ -0,0 +1,111 @@
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from quart import Quart
+from quart_jwt_extended import JWTManager, create_access_token
+from tortoise import Tortoise
+
+from blueprints import integrations
+from blueprints.users.models import User
+from blueprints.ynab import tools
+from blueprints.ynab.sync import tool_data_freshness
+
+
+async def test_stale_ynab_status_hides_provider_error(monkeypatch):
+    monkeypatch.setattr(integrations, "ynab_configured", lambda: True)
+    monkeypatch.setattr(
+        integrations,
+        "_current_state",
+        AsyncMock(
+            return_value=SimpleNamespace(
+                last_synced_at=datetime.now(timezone.utc) - timedelta(hours=1),
+                last_error="Authorization: secret-provider-key",
+            )
+        ),
+    )
+    result = await integrations._ynab_status()
+    assert result["status"] == "stale"
+    assert result["last_success_at"]
+    assert "secret-provider-key" not in str(result)
+
+
+async def test_fresh_ynab_status_is_ready(monkeypatch):
+    monkeypatch.setattr(integrations, "ynab_configured", lambda: True)
+    monkeypatch.setattr(
+        integrations,
+        "_current_state",
+        AsyncMock(
+            return_value=SimpleNamespace(
+                last_synced_at=datetime.now(timezone.utc), last_error=None
+            )
+        ),
+    )
+    assert (await integrations._ynab_status())["status"] == "ready"
+
+
+async def test_stale_tool_data_warns_model_and_carries_artifact_provenance():
+    @tools.ynab_tool_errors
+    async def result():
+        tool_data_freshness.set(
+            {"system": "ynab", "as_of": "2026-10-01T00:00:00+00:00", "stale": True}
+        )
+        return "Total: $100", {"kind": "ynab.summary", "data": {"total": 100}}
+
+    text, artifact = await result()
+    assert "cached budget data is stale" in text
+    assert "2026-10-01" in text
+    assert artifact["source_ref"]["stale"] is True
+    assert artifact["data"] == {"total": 100}
+
+
+async def test_tool_freshness_does_not_leak_from_previous_call():
+    tool_data_freshness.set({"system": "ynab", "as_of": "old", "stale": True})
+
+    @tools.ynab_tool_errors
+    async def result():
+        return "Dates must be YYYY-MM-DD."
+
+    assert await result() == "Dates must be YYYY-MM-DD."
+
+
+@pytest.fixture
+async def status_app(monkeypatch):
+    await Tortoise.init(
+        db_url="sqlite://:memory:", modules={"models": ["blueprints.users.models"]}
+    )
+    await Tortoise.generate_schemas()
+    user = await User.create(username="owner", email="owner@example.com")
+    app = Quart(__name__)
+    app.config["JWT_SECRET_KEY"] = "test-secret"
+    JWTManager(app)
+    app.register_blueprint(integrations.integrations_blueprint)
+    monkeypatch.setattr(
+        integrations,
+        "integration_statuses",
+        AsyncMock(
+            return_value=[
+                {
+                    "name": "YNAB",
+                    "status": "stale",
+                    "message": "Cached data",
+                    "last_success_at": None,
+                }
+            ]
+        ),
+    )
+    async with app.app_context():
+        token = create_access_token(identity=str(user.id))
+    yield app, token
+    await Tortoise.close_connections()
+
+
+async def test_status_requires_access_token(status_app):
+    app, token = status_app
+    assert (await app.test_client().get("/api/integrations/")).status_code == 401
+    response = await app.test_client().get(
+        "/api/integrations/", headers={"Authorization": f"Bearer {token}"}
+    )
+    assert response.status_code == 200
+    assert (await response.get_json())["integrations"][0]["status"] == "stale"

tests/unit/test_job_reliability.py

@@ -0,0 +1,690 @@
+"""Real SQLite ORM delivery claims; providers/runtime are replaced locally."""
+
+import asyncio
+import os
+import sys
+from datetime import timedelta
+from unittest.mock import AsyncMock
+
+import pytest
+from quart import Quart
+from tortoise import Tortoise
+
+from blueprints.cron.models import CronJob
+from blueprints.cron.scheduler import execute_job
+from blueprints.scheduled_messages import delivery
+from blueprints.scheduled_messages.models import (
+    JobAttempt,
+    JobDelivery,
+    MessageChannel,
+    MessageStatus,
+    Recurrence,
+    ScheduledMessage,
+)
+from blueprints.scheduled_messages.scheduler import (
+    _schedule_next_occurrence,
+    process_due_messages,
+    process_message,
+    run_id,
+)
+from blueprints.users.models import User
+
+MODULES = [
+    "blueprints.users.models",
+    "blueprints.scheduled_messages.models",
+    "blueprints.cron.models",
+]
+
+
+@pytest.fixture
+async def db(tmp_path):
+    path = tmp_path / "jobs.sqlite3"
+    await Tortoise.init(db_url=f"sqlite://{path}", modules={"models": MODULES})
+    await Tortoise.generate_schemas()
+    yield path
+    await Tortoise.close_connections()
+
+
+@pytest.fixture
+async def owner(db):
+    return await User.create(username="owner", email="owner@example.com")
+
+
+@pytest.fixture
+async def message(owner):
+    return await ScheduledMessage.create(
+        recipient="recipient",
+        channel=MessageChannel.EMAIL,
+        subject="subject",
+        content="prompt",
+        scheduled_at=delivery.now() - timedelta(minutes=1),
+        created_by=owner,
+    )
+
+
+@pytest.fixture
+async def job(owner):
+    return await CronJob.create(
+        name="cron",
+        prompt="prompt",
+        recipient="recipient",
+        channel=MessageChannel.EMAIL,
+        created_by=owner,
+    )
+
+
+@pytest.fixture
+async def providers(monkeypatch):
+    send = AsyncMock()
+    agent = AsyncMock(return_value="response")
+    monkeypatch.setattr(delivery, "send", send)
+    monkeypatch.setattr(delivery, "agent_content", agent)
+    return send, agent
+
+
+async def test_atomic_concurrent_scheduled_claims(message, providers):
+    results = await asyncio.gather(*[process_message(message) for _ in range(8)])
+    assert results.count("sent") == 1
+    assert providers[0].await_count == 1
+    assert await JobAttempt.all().count() == 1
+    await message.refresh_from_db()
+    assert message.status == MessageStatus.SENT
+
+
+async def test_claims_across_independent_processes(db):
+    # Both processes use their own SQLite connection and ORM registry.
+    code = """
+import asyncio, sys
+from tortoise import Tortoise
+from blueprints.scheduled_messages.delivery import claim
+async def main():
+    await Tortoise.init(db_url="sqlite://" + sys.argv[1], modules={"models": ["blueprints.users.models", "blueprints.scheduled_messages.models", "blueprints.cron.models"]})
+    result = await claim("cron", "same-job", "same-run")
+    print("won" if result else "lost")
+    await Tortoise.close_connections()
+asyncio.run(main())
+"""
+
+    async def worker():
+        p = await asyncio.create_subprocess_exec(
+            sys.executable,
+            "-c",
+            code,
+            str(db),
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env=os.environ.copy(),
+        )
+        out, err = await p.communicate()
+        assert p.returncode == 0, err.decode()
+        return out.decode().strip()
+
+    assert sorted(await asyncio.gather(worker(), worker())) == ["lost", "won"]
+    assert await JobAttempt.all().count() == 1
+
+
+async def test_safe_send_failure_explicit_retry(message, providers):
+    providers[0].side_effect = delivery.SafeDeliveryError("rejected before send")
+    assert await process_message(message) == "failed"
+    assert (await process_due_messages())["sent"] == 0
+    providers[0].side_effect = None
+    await message.refresh_from_db()
+    assert await process_message(message, retry=True) == "sent"
+    assert [a.status for a in await JobAttempt.all().order_by("started_at")] == [
+        "failed",
+        "sent",
+    ]
+    assert providers[0].await_count == 2
+
+
+async def test_ambiguous_provider_failure_never_resends(message, providers):
+    providers[0].side_effect = TimeoutError("may have delivered")
+    assert await process_message(message) == "uncertain"
+    await message.refresh_from_db()
+    assert await process_message(message, retry=True) == "skipped"
+    await process_due_messages()
+    assert providers[0].await_count == 1
+    assert (await JobAttempt.first()).status == "uncertain"
+
+
+@pytest.mark.parametrize("phase", ["processing", "sending"])
+async def test_abandoned_claim_is_uncertain(message, providers, phase):
+    claimed = await delivery.claim("scheduled", message.id, run_id(message))
+    if phase == "sending":
+        await delivery.transition(claimed, "sending")
+    await JobDelivery.filter(id=claimed.id).update(
+        claimed_at=delivery.now() - timedelta(hours=2)
+    )
+    await ScheduledMessage.filter(id=message.id).update(status=MessageStatus.PROCESSING)
+    await process_due_messages()
+    await message.refresh_from_db()
+    assert message.status == MessageStatus.UNCERTAIN
+    assert (await JobAttempt.first()).status == "uncertain"
+    providers[0].assert_not_awaited()
+
+
+async def test_crash_after_provider_send_never_resends(message, providers):
+    class WorkerCrash(BaseException):
+        pass
+
+    async def delivered_then_crash(*args):
+        raise WorkerCrash()
+
+    providers[0].side_effect = delivered_then_crash
+    with pytest.raises(WorkerCrash):
+        await process_message(message)
+    assert (await JobDelivery.first()).status == "sending"
+    await JobDelivery.all().update(claimed_at=delivery.now() - timedelta(hours=2))
+    await process_due_messages()
+    await message.refresh_from_db()
+    assert message.status == MessageStatus.UNCERTAIN
+    assert providers[0].await_count == 1
+
+
+async def test_agent_failure_safe_retry_uses_owner(message, providers):
+    message.use_agent = True
+    await message.save()
+    providers[1].side_effect = ValueError("no text")
+    assert await process_message(message) == "failed"
+    providers[0].assert_not_awaited()
+    providers[1].side_effect = None
+    await message.refresh_from_db()
+    assert await process_message(message, retry=True) == "sent"
+    args = providers[1].await_args.args
+    assert args[0].id == message.created_by_id
+    assert args[-1] == "scheduled"
+
+
+async def test_recurrence_deduplicates_and_skips_missed_occurrences(message, providers):
+    message.recurrence = Recurrence.DAILY
+    message.scheduled_at = delivery.now() - timedelta(days=3)
+    await message.save()
+    assert await process_message(message) == "sent"
+    await asyncio.gather(*[_schedule_next_occurrence(message) for _ in range(5)])
+    assert await ScheduledMessage.all().count() == 2
+    successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
+    assert successor.scheduled_at > delivery.now()
+    assert successor.series_id == message.id
+    await process_due_messages()
+    assert await ScheduledMessage.all().count() == 2
+
+
+async def test_cancelled_series_does_not_regenerate(message, providers):
+    message.recurrence = Recurrence.DAILY
+    await message.save()
+    await process_message(message)
+    successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
+    await ScheduledMessage.filter(id=successor.id).update(
+        status=MessageStatus.CANCELLED
+    )
+    await _schedule_next_occurrence(message)
+    await process_due_messages()
+    assert await ScheduledMessage.all().count() == 2
+    assert providers[0].await_count == 1
+
+
+async def test_legacy_sent_rows_are_not_forked_into_new_series(owner, providers):
+    # Rows sent before delivery claims existed have no successor links or
+    # delivery records; the repair pass must leave them alone.
+    common = dict(
+        recipient="recipient",
+        channel=MessageChannel.EMAIL,
+        subject="subject",
+        content="prompt",
+        recurrence=Recurrence.DAILY,
+        created_by=owner,
+    )
+    for days in (2, 1):
+        await ScheduledMessage.create(
+            **common,
+            status=MessageStatus.SENT,
+            scheduled_at=delivery.now() - timedelta(days=days),
+        )
+    await ScheduledMessage.create(
+        **common, scheduled_at=delivery.now() + timedelta(hours=12)
+    )
+    await process_due_messages()
+    assert await ScheduledMessage.filter(status=MessageStatus.PENDING).count() == 1
+    assert await ScheduledMessage.all().count() == 3
+    providers[0].assert_not_awaited()
+
+
+async def test_claimed_send_without_successor_is_repaired(message, providers):
+    message.recurrence = Recurrence.DAILY
+    await message.save()
+    assert await process_message(message) == "sent"
+    # Simulate a crash after the send committed but before the successor existed.
+    await ScheduledMessage.filter(previous_occurrence_id=message.id).delete()
+    await process_due_messages()
+    assert await ScheduledMessage.filter(previous_occurrence_id=message.id).exists()
+
+
+async def test_cancellation_during_agent_prevents_delivery(message, providers):
+    message.use_agent = True
+    await message.save()
+
+    async def cancel(*args):
+        await ScheduledMessage.filter(id=message.id).update(
+            status=MessageStatus.CANCELLED
+        )
+        return "response"
+
+    providers[1].side_effect = cancel
+    assert await process_message(message) == "cancelled"
+    providers[0].assert_not_awaited()
+    assert (await JobDelivery.first()).status == "cancelled"
+
+
+async def test_duplicate_cron_triggers_scoped_to_run(job, providers):
+    results = await asyncio.gather(
+        *[execute_job(job, "occurrence-1") for _ in range(6)]
+    )
+    assert sum(bool(r.get("ok")) for r in results) == 1
+    assert providers[0].await_count == 1
+    assert await execute_job(job, "occurrence-2") == {
+        "ok": True,
+        "status": "sent",
+        "error": None,
+    }
+    assert await JobAttempt.all().count() == 2
+    assert providers[1].await_args.args[0].id == job.created_by_id
+    assert providers[1].await_args.args[-1] == "cron"
+
+
+async def test_legacy_cron_has_no_implicit_admin(job, providers):
+    job.created_by_id = None
+    await job.save()
+    result = await execute_job(job, "legacy")
+    assert result["status"] == "failed"
+    assert "explicit" in result["error"]
+    providers[1].assert_not_awaited()
+    providers[0].assert_not_awaited()
+
+
+async def test_cron_failure_retry_and_ambiguous_rejection(job, providers):
+    providers[1].side_effect = ValueError("agent failed")
+    assert (await execute_job(job, "run"))["status"] == "failed"
+    assert (await execute_job(job, "run"))["skipped"]
+    providers[1].side_effect = None
+    providers[0].side_effect = TimeoutError("ambiguous")
+    assert (await execute_job(job, "run", retry=True))["status"] == "uncertain"
+    assert (await execute_job(job, "run", retry=True))["skipped"]
+    assert await JobAttempt.all().count() == 2
+    assert providers[0].await_count == 1
+
+
+async def test_disabled_cron_prevents_send_after_agent(job, providers):
+    async def disable(*args):
+        await CronJob.filter(id=job.id).update(enabled=False)
+        return "response"
+
+    providers[1].side_effect = disable
+    assert (await execute_job(job, "run"))["status"] == "cancelled"
+    providers[0].assert_not_awaited()
+
+
+async def test_history_records_attempts(message, providers):
+    await process_message(message)
+    result = await delivery.history("scheduled", message.id)
+    assert result[0]["status"] == "sent"
+    assert not result[0]["retryable"]
+    assert result[0]["attempts"][0]["finished_at"]
+
+
+@pytest.fixture
+async def api(owner, monkeypatch):
+    from quart_jwt_extended import JWTManager, create_access_token
+    from blueprints.cron import cron_blueprint
+    from blueprints.scheduled_messages import scheduled_messages_blueprint
+
+    app = Quart(__name__)
+    app.config["JWT_SECRET_KEY"] = "test-secret" * 8
+    JWTManager(app)
+    app.register_blueprint(cron_blueprint)
+    app.register_blueprint(scheduled_messages_blueprint)
+    monkeypatch.setenv("CRON_SECRET", "cron-secret")
+    owner.ldap_groups = ["lldap_admin"]
+    await owner.save()
+    async with app.app_context():
+        token = create_access_token(identity=str(owner.id))
+    return app.test_client(), {"Authorization": f"Bearer {token}"}
+
+
+async def test_trigger_auth_and_required_cron_identity(job, api, providers):
+    client, _ = api
+    url = f"/api/cron/{job.id}/trigger"
+    assert (await client.post(url, json={"run_id": "run"})).status_code == 401
+    assert (
+        await client.post(url, headers={"X-Cron-Secret": "cron-secret"})
+    ).status_code == 400
+    headers = {"X-Cron-Secret": "cron-secret", "Idempotency-Key": "run"}
+    assert (await client.post(url, headers=headers)).status_code == 200
+    assert (await (await client.post(url, headers=headers)).get_json())["skipped"]
+    assert providers[0].await_count == 1
+
+
+async def test_admin_history_retry_and_failed_reschedule(message, providers, api):
+    client, headers = api
+    url = f"/api/scheduled-messages/{message.id}"
+    providers[0].side_effect = delivery.SafeDeliveryError("not delivered")
+    await process_message(message)
+    response = await client.put(
+        url,
+        headers=headers,
+        json={"scheduled_at": (delivery.now() + timedelta(hours=1)).isoformat()},
+    )
+    assert response.status_code == 200
+    assert (await response.get_json())["status"] == "failed"
+    assert (await client.post(url + "/retry")).status_code == 401
+    providers[0].side_effect = None
+    assert (await client.post(url + "/retry", headers=headers)).status_code == 200
+    data = await (await client.get(url + "/deliveries", headers=headers)).get_json()
+    assert len(data[0]["attempts"]) == 2
+
+
+async def test_api_rejects_ambiguous_retry(message, providers, api):
+    client, headers = api
+    providers[0].side_effect = TimeoutError("unknown")
+    await process_message(message)
+    assert (
+        await client.post(
+            f"/api/scheduled-messages/{message.id}/retry", headers=headers
+        )
+    ).status_code == 409
+    assert providers[0].await_count == 1
+
+
+async def test_api_cancel_recurring_series(message, providers, api):
+    client, headers = api
+    message.recurrence = Recurrence.DAILY
+    await message.save()
+    await process_message(message)
+    successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
+    response = await client.put(
+        f"/api/scheduled-messages/{message.id}",
+        headers=headers,
+        json={"status": "cancelled"},
+    )
+    assert response.status_code == 200
+    await successor.refresh_from_db()
+    assert successor.status == MessageStatus.CANCELLED
+    await process_due_messages()
+    assert await ScheduledMessage.all().count() == 2
+
+
+async def test_cancel_keeps_delivered_status_and_stops_series(message, providers, api):
+    client, headers = api
+    message.recurrence = Recurrence.DAILY
+    await message.save()
+    await process_message(message)
+    response = await client.put(
+        f"/api/scheduled-messages/{message.id}",
+        headers=headers,
+        json={"status": "cancelled"},
+    )
+    assert response.status_code == 200
+    await message.refresh_from_db()
+    assert message.status == MessageStatus.SENT  # delivered stays delivered
+    successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
+    assert successor.status == MessageStatus.CANCELLED
+
+
+async def test_cancel_during_send_does_not_claim_cancelled(message, providers, api):
+    client, headers = api
+    message.recurrence = Recurrence.DAILY
+    await message.save()
+    in_flight, release = asyncio.Event(), asyncio.Event()
+
+    async def slow_send(*args):
+        in_flight.set()
+        await release.wait()
+
+    providers[0].side_effect = slow_send
+    task = asyncio.create_task(process_message(message))
+    await in_flight.wait()
+    response = await client.put(
+        f"/api/scheduled-messages/{message.id}",
+        headers=headers,
+        json={"status": "cancelled"},
+    )
+    assert response.status_code == 200
+    release.set()
+    assert await task == "sent"
+    await message.refresh_from_db()
+    assert message.status == MessageStatus.SENT
+    # The series stops even though nothing pending existed when cancelled.
+    await process_due_messages()
+    assert not await ScheduledMessage.filter(
+        status=MessageStatus.PENDING, deleted_at__isnull=True
+    ).exists()
+
+
+async def test_deleting_edited_occurrence_does_not_revive_series(
+    message, providers, api
+):
+    client, headers = api
+    message.recurrence = Recurrence.DAILY
+    await message.save()
+    await process_message(message)
+    successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
+    url = f"/api/scheduled-messages/{successor.id}"
+    assert (
+        await client.put(url, headers=headers, json={"recurrence": "none"})
+    ).status_code == 200
+    assert (await client.delete(url, headers=headers)).status_code == 200
+    await process_due_messages()
+    assert not await ScheduledMessage.filter(
+        status=MessageStatus.PENDING, deleted_at__isnull=True
+    ).exists()
+
+
+async def test_api_assign_legacy_owner(job, api, providers, owner):
+    job.created_by_id = None
+    await job.save()
+    client, headers = api
+    response = await client.put(
+        f"/api/cron/{job.id}", headers=headers, json={"created_by": str(owner.id)}
+    )
+    assert response.status_code == 200
+    await job.refresh_from_db()
+    assert (await execute_job(job, "run"))["ok"]
+
+
+async def test_due_snapshot_does_not_send_rescheduled_message(message, providers):
+    await ScheduledMessage.filter(id=message.id).update(
+        scheduled_at=delivery.now() + timedelta(hours=1), content="edited"
+    )
+    assert await process_message(message) == "skipped"
+    providers[0].assert_not_awaited()
+
+
+async def test_claim_loads_current_content(message, providers):
+    await ScheduledMessage.filter(id=message.id).update(content="edited")
+    assert await process_message(message) == "sent"
+    assert providers[0].await_args.args[2] == "edited"
+
+
+async def test_delete_recurring_hides_tombstone_and_prevents_repair(
+    message, providers, api
+):
+    message.recurrence = Recurrence.DAILY
+    await message.save()
+    await process_message(message)
+    successor = await ScheduledMessage.get(previous_occurrence_id=message.id)
+    client, headers = api
+    response = await client.delete(
+        f"/api/scheduled-messages/{successor.id}", headers=headers
+    )
+    assert response.status_code == 200
+    rows = await (
+        await client.get("/api/scheduled-messages/", headers=headers)
+    ).get_json()
+    assert [r["id"] for r in rows] == [str(message.id)]
+    await process_due_messages()
+    assert await ScheduledMessage.all().count() == 2
+    assert (await ScheduledMessage.get(id=successor.id)).deleted_at is not None
+
+
+@pytest.mark.parametrize(
+    "status, safe", [(200, None), (401, True), (429, True), (500, False)]
+)
+async def test_email_provider_acknowledgement(monkeypatch, status, safe):
+    import httpx
+
+    monkeypatch.setenv("MAILGUN_API_KEY", "test")
+    monkeypatch.setenv("MAILGUN_DOMAIN", "test.invalid")
+    transport = httpx.MockTransport(
+        lambda request: httpx.Response(status, json={"id": "test"})
+    )
+    original = httpx.AsyncClient
+    monkeypatch.setattr(httpx, "AsyncClient", lambda: original(transport=transport))
+    if safe is None:
+        await delivery.send(MessageChannel.EMAIL, "recipient", "body", "subject")
+    else:
+        with pytest.raises(
+            delivery.SafeDeliveryError if safe else httpx.HTTPStatusError
+        ):
+            await delivery.send(MessageChannel.EMAIL, "recipient", "body", "subject")
+
+
+async def test_missing_email_configuration_is_safe_failure(monkeypatch):
+    monkeypatch.delenv("MAILGUN_API_KEY", raising=False)
+    monkeypatch.delenv("MAILGUN_DOMAIN", raising=False)
+    with pytest.raises(delivery.SafeDeliveryError):
+        await delivery.send(MessageChannel.EMAIL, "recipient", "body")
+
+
+async def test_agent_adapter_uses_shared_runtime(owner, monkeypatch):
+    import types
+
+    runtime = types.ModuleType("blueprints.conversation.runtime")
+    invoke = AsyncMock(return_value={"messages": [{"content": "answer"}]})
+    runtime.invoke_agent_for_user = invoke
+    monkeypatch.setitem(sys.modules, "blueprints.conversation.runtime", runtime)
+    assert (
+        await delivery.agent_content(owner, "query", "thread", "scheduled") == "answer"
+    )
+    invoke.assert_awaited_once_with(owner, "query", "thread", channel="scheduled")
+
+
+async def test_migration_snapshot_preserves_prior_models_and_describes_new_models(db):
+    import importlib.util
+    from pathlib import Path
+    from aerich.utils import compress_dict, decompress_dict, get_models_describe
+
+    def load(path):
+        spec = importlib.util.spec_from_file_location("test_migration", path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        return module
+
+    migration_dir = Path(__file__).resolve().parents[2] / "migrations" / "models"
+    previous = load(migration_dir / "19_20261012225640_add_fk_indexes.py")
+    migration = load(migration_dir / "20_20261012231017_application_foundation.py")
+    state = decompress_dict(migration.MODELS_STATE)
+    owned = {
+        "models." + name
+        for name in ("ScheduledMessage", "CronJob", "JobDelivery", "JobAttempt")
+    }
+    current = decompress_dict(compress_dict(get_models_describe("models")))
+    for name in owned:
+        assert state[name] == current[name]
+    assert "models.User" in decompress_dict(previous.MODELS_STATE)
+    sql = await migration.upgrade(None)
+    assert 'UNIQUE ("kind", "job_id", "run_id")' in sql
+    assert '"previous_occurrence_id" UUID UNIQUE' in sql
+    assert '"created_by_id" UUID' in sql
+
+
+@pytest.mark.parametrize(
+    "content, expected",
+    [
+        ([{"type": "text", "text": "answer"}], "answer"),
+        ("<think>private reasoning</think>answer", "answer"),
+    ],
+)
+async def test_agent_adapter_extracts_supported_content(
+    owner, monkeypatch, content, expected
+):
+    import types
+
+    runtime = types.ModuleType("blueprints.conversation.runtime")
+    runtime.invoke_agent_for_user = AsyncMock(
+        return_value={"messages": [{"content": content}]}
+    )
+    monkeypatch.setitem(sys.modules, "blueprints.conversation.runtime", runtime)
+    assert await delivery.agent_content(owner, "query", "thread", "cron") == expected
+
+
+@pytest.mark.parametrize(
+    "path, body",
+    [
+        ("/api/scheduled-messages/", ["array"]),
+        ("/api/scheduled-messages/", {"recipient": 5, "channel": "email"}),
+        ("/api/cron/", ["array"]),
+        ("/api/cron/", {"name": 7, "prompt": "p", "recipient": "r"}),
+    ],
+)
+async def test_malformed_create_bodies_are_rejected(api, path, body):
+    client, headers = api
+    resp = await client.post(path, headers=headers, json=body)
+    assert resp.status_code == 400
+
+
+async def test_malformed_update_and_trigger_bodies(job, message, api, providers):
+    client, headers = api
+    for path in (f"/api/scheduled-messages/{message.id}", f"/api/cron/{job.id}"):
+        resp = await client.put(path, headers=headers, json={"recipient": ["x"]})
+        assert resp.status_code == 400
+    resp = await client.post(
+        f"/api/cron/{job.id}/trigger",
+        headers={"X-Cron-Secret": "cron-secret"},
+        json=["run"],
+    )
+    assert resp.status_code == 400
+
+
+async def test_update_rejects_null_and_blank_required_fields(job, message, api):
+    client, headers = api
+    for path, field in (
+        (f"/api/scheduled-messages/{message.id}", "recipient"),
+        (f"/api/scheduled-messages/{message.id}", "content"),
+        (f"/api/cron/{job.id}", "name"),
+        (f"/api/cron/{job.id}", "prompt"),
+    ):
+        for value in (None, "   "):
+            resp = await client.put(path, headers=headers, json={field: value})
+            assert resp.status_code == 400, (path, field, value)
+    # subject may be cleared, except where the channel requires it (email).
+    resp = await client.put(
+        f"/api/scheduled-messages/{message.id}", headers=headers, json={"subject": None}
+    )
+    assert resp.status_code == 400
+    resp = await client.put(
+        f"/api/scheduled-messages/{message.id}",
+        headers=headers,
+        json={"subject": None, "channel": "imessage"},
+    )
+    assert resp.status_code == 200
+
+
+async def test_cancel_started_one_off_message_is_rejected(message, providers, api):
+    client, headers = api
+    in_flight, release = asyncio.Event(), asyncio.Event()
+
+    async def slow_send(*args):
+        in_flight.set()
+        await release.wait()
+
+    providers[0].side_effect = slow_send
+    task = asyncio.create_task(process_message(message))
+    await in_flight.wait()
+    response = await client.put(
+        f"/api/scheduled-messages/{message.id}",
+        headers=headers,
+        json={"status": "cancelled"},
+    )
+    assert response.status_code == 409
+    release.set()
+    assert await task == "sent"
+    assert await ScheduledMessage.all().count() == 1  # no tombstone for one-offs

tests/unit/test_memory_retrieval.py

@@ -46,3 +46,34 @@ async def test_backwards_compatible_no_query_and_empty_memories(monkeypatch):
     )
     fetch.return_value = ["Likes tea"]
     assert "Likes tea" in await memory.build_system_prompt_with_memories("user")
+
+
+@pytest.mark.asyncio
+async def test_memory_block_is_identical_across_queries_when_all_fit(monkeypatch):
+    memories = ["Favorite restaurant is Nori", "Cat Simba takes insulin", "Gym is Y"]
+    monkeypatch.setattr(
+        memory, "get_memories_for_user", AsyncMock(return_value=memories)
+    )
+    prompts = {
+        await memory.build_system_prompt_with_memories("user", query=q)
+        for q in ["", "Simba insulin", "gym", "Nori restaurant"]
+    }
+    assert len(prompts) == 1
+    prompt = prompts.pop()
+    assert prompt.index("Nori") < prompt.index("insulin") < prompt.index("Gym")
+
+
+@pytest.mark.asyncio
+async def test_rank_chooses_memories_but_stored_order_is_kept(monkeypatch):
+    filler = "Favorite restaurant is Nori. " * 20
+    memories = ["Likes tea", filler, "Simba insulin dose is two units."]
+    monkeypatch.setattr(
+        memory, "get_memories_for_user", AsyncMock(return_value=memories)
+    )
+    monkeypatch.setattr(memory, "MEMORY_BUDGET_TOKENS", 110)
+    prompt = await memory.build_system_prompt_with_memories(
+        "user", query="Simba insulin"
+    )
+    assert filler not in prompt
+    # Insulin ranks first but is emitted after the newer memory.
+    assert prompt.index("Likes tea") < prompt.index("insulin")

tests/unit/test_obsidian_sync.py

@@ -0,0 +1,188 @@
+"""Tests for the incremental Obsidian sync: off-loop execution and delete SQL."""
+
+import json
+import os
+import threading
+from unittest.mock import patch
+
+import pytest
+
+from blueprints.rag import logic
+
+
+class FakeConnection:
+    def __init__(self, store):
+        self.store = store
+
+    def execute(self, stmt, params=None):
+        self.store.append((str(stmt), params or {}))
+
+    def commit(self):
+        pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        return False
+
+
+class FakeEngine:
+    def __init__(self):
+        self.executed: list[tuple[str, dict]] = []
+
+    def connect(self):
+        return FakeConnection(self.executed)
+
+
+class FakeVectorStore:
+    def __init__(self):
+        self.calls: list[list[str]] = []
+
+    async def aadd_documents(self, documents):
+        self.calls.append([d.page_content for d in documents])
+
+
+@pytest.fixture
+def vault_dir(tmp_path):
+    (tmp_path / "new.md").write_text("# New\nNew note body")
+    (tmp_path / "updated.md").write_text("# Updated\nUpdated note body")
+    with patch.dict(os.environ, {"OBSIDIAN_VAULT_PATH": str(tmp_path)}):
+        yield tmp_path
+
+
+@pytest.fixture
+async def synced(monkeypatch, vault_dir):
+    """Run a sync with a fake vault: one new file, one updated file, one
+    file that's been deleted off disk since the last sync."""
+    updated_path = str(vault_dir / "updated.md")
+    missing_path = str(vault_dir / "gone.md")
+
+    monkeypatch.setattr(
+        logic, "_obsidian_index_cache", {updated_path: 1.0, missing_path: 1.0}
+    )
+
+    engine = FakeEngine()
+    monkeypatch.setattr(logic, "_get_engine", lambda: engine)
+    monkeypatch.setattr(logic, "_get_collection_id", lambda: "collection-uuid")
+
+    def _unexpected_cache_load():
+        raise AssertionError("cache load should be skipped when already populated")
+
+    monkeypatch.setattr(logic, "_load_obsidian_index_cache", _unexpected_cache_load)
+
+    store = FakeVectorStore()
+    monkeypatch.setattr(logic, "_get_vector_store", lambda: store)
+
+    threads: dict[str, threading.Thread] = {}
+
+    real_walk = logic._walk_vault_mtimes
+
+    def recording_walk(service):
+        threads["walk"] = threading.current_thread()
+        return real_walk(service)
+
+    monkeypatch.setattr(logic, "_walk_vault_mtimes", recording_walk)
+
+    real_read = logic._read_obsidian_documents
+
+    def recording_read(service, filepaths, now):
+        threads["read"] = threading.current_thread()
+        return real_read(service, filepaths, now)
+
+    monkeypatch.setattr(logic, "_read_obsidian_documents", recording_read)
+
+    real_delete = logic.delete_documents_by_filepaths
+
+    def recording_delete(filepaths):
+        threads["delete"] = threading.current_thread()
+        return real_delete(filepaths)
+
+    monkeypatch.setattr(logic, "delete_documents_by_filepaths", recording_delete)
+
+    result = await logic.sync_obsidian_documents()
+    return result, threads, engine, store, updated_path, missing_path
+
+
+class TestSyncOffLoop:
+    async def test_counts_match_vault_changes(self, synced):
+        result, *_ = synced
+        assert result == {"added": 1, "updated": 1, "deleted": 1}
+
+    async def test_walk_runs_off_the_main_thread(self, synced):
+        _, threads, *_ = synced
+        assert threads["walk"] is not threading.main_thread()
+
+    async def test_reads_run_off_the_main_thread(self, synced):
+        _, threads, *_ = synced
+        assert threads["read"] is not threading.main_thread()
+
+    async def test_deletes_run_off_the_main_thread(self, synced):
+        _, threads, *_ = synced
+        assert threads["delete"] is not threading.main_thread()
+
+    async def test_embeds_new_and_updated_content(self, synced):
+        _, _, _, store, *_ = synced
+        embedded = {text for call in store.calls for text in call}
+        assert any("New note body" in t for t in embedded)
+        assert any("Updated note body" in t for t in embedded)
+
+    async def test_cache_reflects_add_update_delete(self, synced):
+        result, _, _, _, updated_path, missing_path = synced
+        assert missing_path not in logic._obsidian_index_cache
+        assert updated_path in logic._obsidian_index_cache
+
+    async def test_deletes_use_jsonb_containment_with_bound_params(self, synced):
+        _, _, engine, *_ = synced
+        deletes = [(sql, params) for sql, params in engine.executed if "DELETE" in sql]
+        assert deletes, "expected at least one delete statement"
+        for sql, params in deletes:
+            assert "cmetadata @> CAST(:filter AS jsonb)" in sql
+            assert "->>:" not in sql  # not the old, unindexed text-extraction filter
+            assert "filter" in params
+            parsed_filter = json.loads(params["filter"])
+            assert "filepath" in parsed_filter
+
+
+async def test_delete_failure_keeps_deleted_path_in_cache_and_propagates(
+    monkeypatch, vault_dir
+):
+    """If the batched delete raises, the sync must not have already forgotten
+    the deleted file: otherwise the next sync never retries the delete and
+    its chunks stay searchable forever with no record that they're stale."""
+    updated_path = str(vault_dir / "updated.md")
+    missing_path = str(vault_dir / "gone.md")
+
+    monkeypatch.setattr(
+        logic, "_obsidian_index_cache", {updated_path: 1.0, missing_path: 1.0}
+    )
+
+    def _raise_on_delete(filepaths):
+        raise RuntimeError("db unavailable")
+
+    monkeypatch.setattr(logic, "delete_documents_by_filepaths", _raise_on_delete)
+
+    store = FakeVectorStore()
+    monkeypatch.setattr(logic, "_get_vector_store", lambda: store)
+
+    with pytest.raises(RuntimeError, match="db unavailable"):
+        await logic.sync_obsidian_documents()
+
+    assert missing_path in logic._obsidian_index_cache
+    # The updated file's old chunks were never actually deleted, so its cache
+    # entry must not be bumped to look freshly indexed.
+    assert logic._obsidian_index_cache[updated_path] == 1.0
+    assert store.calls == []
+
+
+async def test_delete_documents_by_metadata_uses_containment(monkeypatch):
+    engine = FakeEngine()
+    monkeypatch.setattr(logic, "_get_engine", lambda: engine)
+    monkeypatch.setattr(logic, "_get_collection_id", lambda: "collection-uuid")
+
+    logic.delete_documents_by_metadata("source", "obsidian")
+
+    assert len(engine.executed) == 1
+    sql, params = engine.executed[0]
+    assert "cmetadata @> CAST(:filter AS jsonb)" in sql
+    assert json.loads(params["filter"]) == {"source": "obsidian"}

tests/unit/test_paperless_indexing.py

@@ -0,0 +1,56 @@
+from unittest.mock import AsyncMock
+
+import pytest
+from langchain_core.documents import Document
+
+from blueprints.rag import logic
+
+
+class FakeVectorStore:
+    def __init__(self, bad_text=None, fail_batches=False):
+        self.calls = []
+        self.bad_text = bad_text
+        self.fail_batches = fail_batches
+
+    async def aadd_documents(self, documents):
+        self.calls.append([d.page_content for d in documents])
+        if self.fail_batches and len(documents) > 1:
+            raise RuntimeError("batch too large")
+        if any(d.page_content == self.bad_text for d in documents):
+            raise RuntimeError("tokenizer error")
+
+
+def _setup(monkeypatch, store, count):
+    docs = [
+        Document(page_content=f"chunk {i}", metadata={"filename": f"doc{i}.pdf"})
+        for i in range(count)
+    ]
+    monkeypatch.setattr(
+        logic, "fetch_documents_from_paperless_ngx", AsyncMock(return_value=docs)
+    )
+    monkeypatch.setattr(logic.text_splitter, "split_documents", lambda d: d)
+    monkeypatch.setattr(logic, "_get_vector_store", lambda: store)
+    monkeypatch.setattr(logic, "EMBED_BATCH_SIZE", 4)
+
+
+async def test_chunks_are_embedded_in_batches(monkeypatch):
+    store = FakeVectorStore()
+    _setup(monkeypatch, store, 10)
+    await logic.index_documents()
+    assert [len(c) for c in store.calls] == [4, 4, 2]
+
+
+async def test_failed_batch_names_the_rejected_chunk(monkeypatch, caplog):
+    store = FakeVectorStore(bad_text="chunk 6")
+    _setup(monkeypatch, store, 10)
+    with pytest.raises(RuntimeError, match="tokenizer error"):
+        await logic.index_documents()
+    assert "Failed to embed chunk 6 from doc6.pdf" in caplog.text
+
+
+async def test_batch_failure_recovers_when_every_chunk_succeeds(monkeypatch):
+    store = FakeVectorStore(fail_batches=True)
+    _setup(monkeypatch, store, 5)
+    await logic.index_documents()
+    singles = [c[0] for c in store.calls if len(c) == 1]
+    assert singles == [f"chunk {i}" for i in range(5)]

tests/unit/test_prompts.py

@@ -0,0 +1,20 @@
+from blueprints.conversation.prompts import SIMBA_SYSTEM_PROMPT
+
+
+def test_prompt_defaults_to_concise_answers():
+    assert "concisely by default" in SIMBA_SYSTEM_PROMPT
+    assert "thorough, detailed answers" not in SIMBA_SYSTEM_PROMPT
+    # Reasoning tokens cost as much decode latency as answer tokens on this
+    # hardware, so the prompt must not nudge toward more deliberation.
+    assert "think things through" not in SIMBA_SYSTEM_PROMPT
+
+
+def test_prompt_keeps_key_behavioral_rules():
+    for phrase in [
+        "you MUST use the simba_search tool",
+        "cite the provided Citation URLs with Markdown links",
+        "never invent source IDs or references",
+        "search before creating to avoid duplicates",
+        "Use save_user_memory when the user explicitly asks",
+    ]:
+        assert phrase in SIMBA_SYSTEM_PROMPT

tests/unit/test_retrieval.py

@@ -0,0 +1,190 @@
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+from uuid import uuid4
+
+import pytest
+from langchain_core.documents import Document
+from langchain_postgres import PGVector
+from quart import Quart
+from quart_jwt_extended import JWTManager, create_access_token
+from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
+
+from blueprints.rag.retrieval import build_filter, citation_url, document_date
+from blueprints.rag import logic, source_api
+from blueprints.conversation.artifacts_util import search_documents_artifact
+
+
+@pytest.mark.parametrize("source", [None, "paperless", "obsidian"])
+@pytest.mark.parametrize(
+    "start,end",
+    [("2026-09-01", "2026-09-30"), ("2026-09-01", None), (None, "2026-09-30")],
+)
+def test_filters_compile_with_actual_pgvector_operator_rules(source, start, end):
+    store = PGVector.__new__(PGVector)
+    store.EmbeddingStore = SimpleNamespace(cmetadata=Column("cmetadata", JSONB))
+    expression = store._create_filter_clause(build_filter(source, start, end))
+    assert expression is not None
+
+
+@pytest.mark.parametrize(
+    "start,end",
+    [
+        ("20260901", None),
+        ("bad", None),
+        ("2026-10-01", "2026-09-01"),
+        (None, "9999-12-31"),
+    ],
+)
+def test_date_filters_validate_before_vector_lookup(start, end):
+    with pytest.raises(ValueError):
+        build_filter("obsidian", start, end)
+
+
+async def test_filtered_query_preserves_citations_and_excludes_unknown_dates(
+    monkeypatch,
+):
+    identifier = str(uuid4())
+    docs = [
+        Document(
+            id=identifier,
+            page_content="September groceries plan",
+            metadata={
+                "source": "obsidian",
+                "date": "2026-09-15",
+                "filepath": "notes/budget.md",
+            },
+        ),
+        Document(
+            id=str(uuid4()),
+            page_content="Undated plan",
+            metadata={"source": "obsidian"},
+        ),
+    ]
+    store = SimpleNamespace(amax_marginal_relevance_search=AsyncMock(return_value=docs))
+    monkeypatch.setattr(logic, "_get_vector_store", lambda: store)
+    text, results = await logic.query_vector_store(
+        "budget", source="obsidian", start_date="2026-09-01", end_date="2026-09-30"
+    )
+    assert results == docs[:1]
+    assert f"Citation: /sources/{identifier}" in text
+    assert store.amax_marginal_relevance_search.call_args.kwargs[
+        "filter"
+    ] == build_filter("obsidian", "2026-09-01", "2026-09-30")
+
+
+def test_artifact_citations_identify_exact_retrieved_chunk(monkeypatch):
+    monkeypatch.setenv("OBSIDIAN_VAULT_PATH", "/vault")
+    identifier = str(uuid4())
+    doc = Document(
+        id=identifier,
+        page_content="Savings target",
+        metadata={
+            "filepath": "/vault/notes/budget.md",
+            "h1": "Budget",
+            "h2": "Savings",
+        },
+    )
+    artifact = search_documents_artifact("target", "obsidian", [doc])
+    result = artifact["data"]["results"][0]
+    assert result["url"] == f"/sources/{identifier}"
+    assert result["source_id"] == identifier
+    assert result["path"] == "notes/budget.md"
+    assert result["headings"] == ["Budget", "Savings"]
+
+
+async def test_serialized_chunk_header_is_compact_not_raw_metadata(monkeypatch):
+    identifier = str(uuid4())
+    doc = Document(
+        id=identifier,
+        page_content="Rabies vaccine administered",
+        metadata={
+            "source": "paperless",
+            "filename": "vet_invoice.pdf",
+            "created_date": datetime(2026, 9, 1, tzinfo=timezone.utc).timestamp(),
+            "document_type": "Invoice",
+        },
+    )
+    store = SimpleNamespace(
+        amax_marginal_relevance_search=AsyncMock(return_value=[doc])
+    )
+    monkeypatch.setattr(logic, "_get_vector_store", lambda: store)
+    text, _ = await logic.query_vector_store("vet", source="paperless")
+    assert "{'source'" not in text
+    assert str(doc.metadata) not in text
+    assert "Title: vet_invoice" in text
+    assert "Date: 2026-09-01" in text
+    assert "Source: paperless" in text
+    assert f"Citation: /sources/{identifier}" in text
+    assert "Content: Rabies vaccine administered" in text
+
+
+def test_document_date_and_unknown_citations():
+    assert (
+        document_date(
+            {"created_date": datetime(2026, 9, 1, tzinfo=timezone.utc).timestamp()}
+        )
+        == "2026-09-01"
+    )
+    assert document_date({"date": "bad"}) is None
+    assert citation_url(Document(page_content="no stable ID")) is None
+
+
+async def test_source_read_requires_auth_and_handles_missing_chunk(monkeypatch):
+    app = Quart(__name__)
+    app.config["JWT_SECRET_KEY"] = "test-secret"
+    JWTManager(app)
+    app.register_blueprint(source_api.sources_blueprint)
+    calls = []
+
+    def load(identifier):
+        calls.append(identifier)
+        return {
+            "id": identifier,
+            "source": "paperless",
+            "title": "Vet report",
+            "content": "Retrieved section",
+            "headings": [],
+        }
+
+    monkeypatch.setattr(source_api, "_load_chunk", load)
+    identifier = str(uuid4())
+    assert (
+        await app.test_client().get(f"/api/sources/{identifier}")
+    ).status_code == 401
+    async with app.app_context():
+        token = create_access_token(identity=str(uuid4()))
+    headers = {"Authorization": f"Bearer {token}"}
+    response = await app.test_client().get(
+        f"/api/sources/{identifier}", headers=headers
+    )
+    assert (await response.get_json())["content"] == "Retrieved section"
+    assert response.headers["Cache-Control"] == "no-store"
+    assert (
+        await app.test_client().get("/api/sources/not-an-id", headers=headers)
+    ).status_code == 404
+    assert calls == [identifier]
+
+
+async def test_post_filter_keeps_note_matched_by_any_date_field(monkeypatch):
+    # The database filter matches if any date field is in range; the post-filter
+    # must agree or matching notes silently vanish from results.
+    doc = Document(
+        id=str(uuid4()),
+        page_content="Lease renewal notes",
+        metadata={
+            "source": "obsidian",
+            "date": "2023-05-01",
+            "created_at": "2024-06-12T09:00:00",
+            "filepath": "notes/lease.md",
+        },
+    )
+    store = SimpleNamespace(
+        amax_marginal_relevance_search=AsyncMock(return_value=[doc])
+    )
+    monkeypatch.setattr(logic, "_get_vector_store", lambda: store)
+    _, results = await logic.query_vector_store(
+        "lease", source="obsidian", start_date="2024-06-01", end_date="2024-06-30"
+    )
+    assert results == [doc]

tests/unit/test_stream_stall.py

@@ -26,6 +26,7 @@ async def owner():
             "models": [
                 "blueprints.users.models",
                 "blueprints.conversation.models",
+                "blueprints.conversation.run_models",
                 "blueprints.artifacts.models",
             ]
         },
@@ -61,7 +62,7 @@ def _chunk(run_id, text):
     }
 
 
-def _end(run_id, prompt_tokens=None):
+def _end(run_id, prompt_tokens=None, cached_tokens=None):
     usage = (
         {
             "input_tokens": prompt_tokens,
@@ -71,6 +72,8 @@ def _end(run_id, prompt_tokens=None):
         if prompt_tokens
         else None
     )
+    if usage and cached_tokens is not None:
+        usage["input_token_details"] = {"cache_read": cached_tokens}
     return {
         "event": "on_chat_model_end",
         "run_id": run_id,
@@ -89,12 +92,12 @@ def _final(text):
 async def _run(owner, monkeypatch, script, *, response_timeout=60, direct=False):
     user, conversation = owner
     monkeypatch.setattr(conv, "MODEL_STALL_TIMEOUT", LIMIT)
-    monkeypatch.setattr(conv, "get_agent_for", lambda u: _TimedAgent(script))
+    monkeypatch.setattr(conv.runtime, "get_agent_for", lambda u: _TimedAgent(script))
 
     async def _prompt(user_id, query=""):
         return "system"
 
-    monkeypatch.setattr(conv, "build_system_prompt_with_memories", _prompt)
+    monkeypatch.setattr(conv.runtime, "build_system_prompt_with_memories", _prompt)
 
     app = Quart(__name__)
     app.config["RESPONSE_TIMEOUT"] = response_timeout
@@ -207,7 +210,7 @@ async def test_logs_per_call_prompt_tokens(owner, monkeypatch, caplog):
     script = [
         (0, _start("m1")),
         (0, _chunk("m1", "Hi")),
-        (0, _end("m1", prompt_tokens=27262)),
+        (0, _end("m1", prompt_tokens=27262, cached_tokens=25000)),
         (0, _final("Hi")),
     ]
 
@@ -224,6 +227,7 @@ async def test_logs_per_call_prompt_tokens(owner, monkeypatch, caplog):
     assert summary["max_prompt_tokens"] == 27262
     (call,) = summary["model_calls"]
     assert call["prompt_tokens"] == 27262
+    assert call["cached_prompt_tokens"] == 25000
     assert call["prefill_ms"] is not None and call["ms"] is not None
 
 
@@ -297,18 +301,22 @@ async def test_no_response_is_counted_as_error(owner, monkeypatch):
 async def test_final_persistence_exit_has_exactly_one_outcome(
     owner, monkeypatch, caplog, cancelled
 ):
-    results, durations, writes = [], [], []
-    original = conv.add_message_to_conversation
+    # Answers are persisted by the detached run worker, so the request only
+    # observes the terminal event; cancellation must not escape the worker.
+    from blueprints.conversation import runs
 
-    async def persist(**kwargs):
-        if kwargs["speaker"] == "simba":
-            writes.append(kwargs)
+    results, durations, writes = [], [], []
+    original = runs.add_message_to_conversation
+
+    async def persist(conversation, message, speaker, *args, **kwargs):
+        if speaker == "simba":
+            writes.append(message)
             if cancelled:
                 raise asyncio.CancelledError()
             raise RuntimeError("persistence failed")
-        return await original(**kwargs)
+        return await original(conversation, message, speaker, *args, **kwargs)
 
-    monkeypatch.setattr(conv, "add_message_to_conversation", persist)
+    monkeypatch.setattr(runs, "add_message_to_conversation", persist)
     monkeypatch.setattr(
         conv, "count_result", lambda **kwargs: results.append(kwargs["result"])
     )
@@ -322,13 +330,8 @@ async def test_final_persistence_exit_has_exactly_one_outcome(
         (0, _final("Answer")),
     ]
     with caplog.at_level(logging.INFO):
-        if cancelled:
-            with pytest.raises(asyncio.CancelledError):
-                await _run(owner, monkeypatch, script, direct=True)
-        else:
-            frames = await _run(owner, monkeypatch, script, direct=True)
-            assert frames[-1]["type"] == "error"
-            assert "persistence failed" in frames[-1]["message"]
+        frames = await _run(owner, monkeypatch, script, direct=True)
+    assert frames[-1]["type"] == "error"
     assert len(writes) == 1
     assert results == ["cancelled" if cancelled else "error"]
     assert len(durations) == 1

tests/unit/test_tool_artifacts.py

@@ -521,6 +521,13 @@ async def test_simba_search(monkeypatch):
     assert result["snippet"].startswith("Rabies vaccine administered. ")
     assert len(result["snippet"]) <= SNIPPET_MAX_CHARS
 
+    (model_result,) = art["model_context"]["results"]
+    assert model_result["content"] == PAPERLESS_DOCS[0].page_content
+    assert len(model_result["content"]) > SNIPPET_MAX_CHARS
+    assert model_result["title"] == result["title"]
+    assert model_result["filename"] == result["filename"]
+    assert "snippet" not in model_result
+
 
 async def test_simba_search_empty(monkeypatch):
     monkeypatch.setattr(agents, "query_vector_store", AsyncMock(return_value=("", [])))
@@ -560,6 +567,14 @@ async def test_obsidian_search_notes(obsidian, monkeypatch):
             "date": "2026-01-02",
         }
     ]
+    assert art["model_context"]["results"] == [
+        {
+            "content": "Simba's vet is Dr Smith.",
+            "title": "simba",
+            "path": "pets/simba.md",
+            "date": "2026-01-02",
+        }
+    ]
 
 
 async def test_obsidian_search_notes_error(obsidian, monkeypatch):

tests/unit/test_tool_routing.py

@@ -136,11 +136,7 @@ def test_discovery_recovery_admin_gating_and_isolation(router):
     )
 
 
-@pytest.mark.asyncio
-@pytest.mark.parametrize("primary_fails", [False, True])
-async def test_real_graph_binds_discovered_schemas_and_accounts_same_subset(
-    monkeypatch, primary_fails
-):
+def import_agents():
     # Constructors are stubbed before agents import: no tokens, services or network.
     with (
         patch("langchain_openai.ChatOpenAI"),
@@ -154,7 +150,16 @@ async def test_real_graph_binds_discovered_schemas_and_accounts_same_subset(
             "utils.obsidian_service.ObsidianService", side_effect=ValueError("offline")
         ),
     ):
-        agents = importlib.import_module("blueprints.conversation.agents")
+        return importlib.import_module("blueprints.conversation.agents")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("primary_fails", [False, True])
+async def test_real_graph_binds_discovered_schemas_and_accounts_same_subset(
+    monkeypatch, primary_fails
+):
+    monkeypatch.setenv("TOOL_ROUTING_MODE", "dynamic")
+    agents = import_agents()
     bound_names, primary_names, accounted_names = [], [], []
 
     class FakeModel:
@@ -238,3 +243,90 @@ def test_schema_dict_tools_remain_routable_and_admin_gated():
     router = ToolRouter([extra], admin_tools=[extra])
     assert extra not in router.select([HumanMessage(content="extra")])
     assert extra in router.select([HumanMessage(content="extra")], is_admin=True)
+
+
+def test_static_selection_is_stable_role_gated_and_has_no_discovery(router):
+    queries = ["Review spending", "Find Simba's vet records", "Help", "calendar"]
+    user = [router.select_static() for _ in queries]
+    assert all(selected == user[0] for selected in user)
+    assert [t.name for t in user[0]] == [
+        "get_current_date",
+        "save_user_memory",
+        "ynab_budget_summary",
+        "mealie_get_recipe",
+        "obsidian_read_note",
+        "simba_search",
+        "web_search",
+    ]
+    admin = [t.name for t in router.select_static(is_admin=True)]
+    assert admin == [t.name for t in router.tools]
+    assert "discover_tool_groups" not in admin
+
+
+def test_static_selection_gates_schema_dict_admin_tools():
+    extra = {
+        "type": "function",
+        "function": {
+            "name": "extra",
+            "description": "docs",
+            "parameters": {"type": "object", "properties": {}},
+        },
+    }
+    router = ToolRouter([make_tool("web_search"), extra], admin_tools=[extra])
+    assert extra not in router.select_static()
+    assert extra in router.select_static(is_admin=True)
+
+
+@pytest.mark.asyncio
+async def test_static_graph_binds_full_role_set_every_call(monkeypatch):
+    monkeypatch.delenv("TOOL_ROUTING_MODE", raising=False)
+    agents = import_agents()
+    bound_names = []
+
+    class FakeModel:
+        def bind_tools(self, tools):
+            bound_names.append([t.name for t in tools])
+
+            def respond(messages):
+                if len(bound_names) == 1:
+                    return call("ynab_budget_summary")
+                return AIMessage(content="done")
+
+            return RunnableLambda(respond)
+
+    monkeypatch.setattr(
+        agents,
+        "tools",
+        [
+            make_tool("get_current_date"),
+            make_tool("ynab_budget_summary"),
+            make_tool("get_calendar_events"),
+        ],
+    )
+    monkeypatch.setattr(agents, "llama_chat", None)
+    monkeypatch.setattr(agents, "openai_fallback", FakeModel())
+    monkeypatch.setattr(agents, "build_context", lambda messages, *a, **k: messages)
+    graph = agents.build_agent()
+    # A query matching no keyword group still sees every tool, with no
+    # discovery round trip, and the bound set is identical on every call.
+    result = await graph.ainvoke({"messages": [HumanMessage(content="Help")]})
+    assert result["messages"][-1].content == "done"
+    assert bound_names == [["get_current_date", "ynab_budget_summary"]] * 2
+
+    bound_names.clear()
+    await graph.ainvoke(
+        {"messages": [HumanMessage(content="Help")]},
+        config={"configurable": {"is_admin": True}},
+    )
+    assert bound_names[0] == [
+        "get_current_date",
+        "ynab_budget_summary",
+        "get_calendar_events",
+    ]
+
+
+def test_unknown_routing_mode_fails_explicitly(monkeypatch):
+    agents = import_agents()
+    monkeypatch.setenv("TOOL_ROUTING_MODE", "sometimes")
+    with pytest.raises(ValueError, match="TOOL_ROUTING_MODE"):
+        agents.build_agent()