diff --git a/CLAUDE.md b/CLAUDE.md index df9cc80..2474783 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,26 +11,28 @@ SimbaRAG is a RAG (Retrieval-Augmented Generation) conversational AI system for ### Development ```bash -# Start environment -docker compose up --build +# Start environment (raggr runs as raggr-blue/raggr-green behind compose profiles) +docker compose --profile blue up --build -# View logs -docker compose logs -f raggr +# View logs (Makefile resolves the active color) +make logs ``` ### Database Migrations (Aerich/Tortoise ORM) ```bash # Generate migration (must run in Docker with DB access) -docker compose exec raggr aerich migrate --name describe_change +make migrate-new # Apply migrations (auto-runs on startup, manual if needed) -docker compose exec raggr aerich upgrade +make migrate # View migration history -docker compose exec raggr aerich history +make migrate-history ``` +Migrations must be backward-compatible: blue-green deploys run `aerich upgrade` in the new container while the old code still serves. + ### Frontend ```bash @@ -42,9 +44,10 @@ yarn dev # Dev server (rarely needed, backend serves frontend) ### Production +Zero-downtime blue-green deploy: `raggr-blue` (port 8080) and `raggr-green` (port 8081) alternate behind Caddy health checks. See `docs/deployment.md`. + ```bash -docker compose build raggr -docker compose up -d +make redeploy # git pull + scripts/deploy.sh (build new color, wait healthy, stop old) ``` ## Architecture @@ -53,7 +56,7 @@ docker compose up -d ┌─────────────────────────────────────────────────────────────┐ │ Docker Compose │ ├─────────────────────────────────────────────────────────────┤ -│ raggr (port 8080) │ postgres (port 5432) │ +│ raggr-blue/green (8080/8081) │ postgres (port 5432) │ │ ├── Quart backend │ PostgreSQL 16 + pgvector│ │ └── React frontend (served) │ │ └─────────────────────────────────────────────────────────────┘ diff --git a/Makefile b/Makefile index eef5560..2460b27 100644 --- a/Makefile +++ b/Makefile @@ -1,36 +1,65 @@ -.PHONY: deploy redeploy build up down restart logs migrate migrate-new frontend test +.PHONY: deploy redeploy build up down restart logs migrate migrate-new migrate-history frontend test test-cov reindex reindex-obsidian + +# Blue-green: detect the running color (default blue when nothing is running) +ACTIVE_COLOR := $(shell docker compose ps --status running --services 2>/dev/null | grep -oE 'raggr-(blue|green)' | head -1 | cut -d- -f2) +ifeq ($(ACTIVE_COLOR),) +ACTIVE_COLOR := blue +endif +RAGGR := raggr-$(ACTIVE_COLOR) +# Both profiles enabled for commands that only touch existing containers +# (never use this with `up`, which would start both colors) +COMPOSE := docker compose --profile blue --profile green # Build and deploy deploy: build up +# Zero-downtime blue-green swap (see scripts/deploy.sh) redeploy: - git pull && $(MAKE) down && $(MAKE) up + git pull && ./scripts/deploy.sh build: - docker compose build raggr + docker compose --profile $(ACTIVE_COLOR) build $(RAGGR) up: - docker compose up -d + docker compose --profile $(ACTIVE_COLOR) up -d down: - docker compose down + $(COMPOSE) down restart: - docker compose restart raggr + $(COMPOSE) restart $(RAGGR) logs: - docker compose logs -f raggr + $(COMPOSE) logs -f $(RAGGR) + +# Vector store reindexing +reindex: + $(COMPOSE) exec $(RAGGR) python -c "\ + import asyncio; \ + from blueprints.rag.logic import delete_all_documents, index_documents, index_obsidian_documents; \ + delete_all_documents(); \ + asyncio.run(index_documents()); \ + asyncio.run(index_obsidian_documents()); \ + print('Reindex complete')" + +reindex-obsidian: + $(COMPOSE) exec $(RAGGR) python -c "\ + import asyncio; \ + from blueprints.rag.logic import delete_documents_by_metadata, index_obsidian_documents; \ + delete_documents_by_metadata('source', 'obsidian'); \ + asyncio.run(index_obsidian_documents()); \ + print('Obsidian reindex complete')" # Database migrations migrate: - docker compose exec raggr aerich upgrade + $(COMPOSE) exec $(RAGGR) aerich upgrade migrate-new: @read -p "Migration name: " name; \ - docker compose exec raggr aerich migrate --name $$name + $(COMPOSE) exec $(RAGGR) aerich migrate --name $$name migrate-history: - docker compose exec raggr aerich history + $(COMPOSE) exec $(RAGGR) aerich history # Tests test: diff --git a/README.md b/README.md index df8b779..96757fb 100644 --- a/README.md +++ b/README.md @@ -127,8 +127,8 @@ cp .env.example .env # Development (local PostgreSQL only) docker compose -f docker-compose.dev.yml up -d -# Or full Docker deployment -docker compose up -d +# Or full Docker deployment (app runs as raggr-blue; see docs/deployment.md) +docker compose --profile blue up -d ``` 4. **Access the application** diff --git a/app.py b/app.py index 6d4a8c1..c564b8b 100644 --- a/app.py +++ b/app.py @@ -144,6 +144,16 @@ async def serve_react_app(path): return await render_template("index.html") +# Health check used by the Docker healthcheck and Caddy's active health checks +@app.route("/api/health", methods=["GET"]) +async def health(): + try: + await Tortoise.get_connection("default").execute_query("SELECT 1") + except Exception: + return jsonify({"status": "unhealthy"}), 503 + return jsonify({"status": "ok"}) + + @app.route("/api/messages", methods=["GET"]) @jwt_refresh_token_required async def get_messages(): diff --git a/docker-compose.yml b/docker-compose.yml index 4efb79b..c078f00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,80 @@ version: "3.8" +x-raggr-common: &raggr-common + build: + context: . + dockerfile: Dockerfile + image: torrtle/simbarag:latest + environment: + - PAPERLESS_TOKEN=${PAPERLESS_TOKEN} + - BASE_URL=${BASE_URL} + - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} + - OPENAI_API_KEY=${OPENAI_API_KEY} + - EMBEDDING_SERVER_URL=${EMBEDDING_SERVER_URL} + - EMBEDDING_MODEL_NAME=${EMBEDDING_MODEL_NAME} + - JWT_SECRET_KEY=${JWT_SECRET_KEY} + - LLAMA_SERVER_URL=${LLAMA_SERVER_URL} + - LLAMA_MODEL_NAME=${LLAMA_MODEL_NAME} + - OIDC_ISSUER=${OIDC_ISSUER} + - OIDC_CLIENT_ID=${OIDC_CLIENT_ID} + - OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET} + - OIDC_REDIRECT_URI=${OIDC_REDIRECT_URI} + - OIDC_USE_DISCOVERY=${OIDC_USE_DISCOVERY:-true} + - DATABASE_URL=${DATABASE_URL:-postgres://raggr:changeme@postgres:5432/raggr} + - TAVILY_API_KEY=${TAVILIY_API_KEY} + - YNAB_ACCESS_TOKEN=${YNAB_ACCESS_TOKEN} + - YNAB_BUDGET_ID=${YNAB_BUDGET_ID} + - MEALIE_BASE_URL=${MEALIE_BASE_URL} + - MEALIE_API_TOKEN=${MEALIE_API_TOKEN} + - TWILIO_ACCOUNT_SID=${TWILIO_ACCOUNT_SID} + - TWILIO_AUTH_TOKEN=${TWILIO_AUTH_TOKEN} + - TWILIO_WHATSAPP_NUMBER=${TWILIO_WHATSAPP_NUMBER} + - ALLOWED_WHATSAPP_NUMBERS=${ALLOWED_WHATSAPP_NUMBERS} + - TWILIO_SIGNATURE_VALIDATION=${TWILIO_SIGNATURE_VALIDATION:-true} + - TWILIO_WEBHOOK_URL=${TWILIO_WEBHOOK_URL:-} + - OBSIDIAN_EMAIL=${OBSIDIAN_EMAIL} + - OBSIDIAN_PASSWORD=${OBSIDIAN_PASSWORD} + - OBSIDIAN_VAULT_ID=${OBSIDIAN_VAULT_ID} + - OBSIDIAN_E2E_PASSWORD=${OBSIDIAN_E2E_PASSWORD} + - OBSIDIAN_DEVICE_NAME=${OBSIDIAN_DEVICE_NAME} + - OBSIDIAN_CONTINUOUS_SYNC=${OBSIDIAN_CONTINUOUS_SYNC:-false} + - OBSIDIAN_VAULT_PATH=${OBSIDIAN_VAULT_PATH:-/app/data/obsidian} + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL} + - S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID} + - S3_SECRET_ACCESS_KEY=${S3_SECRET_ACCESS_KEY} + - S3_BUCKET_NAME=${S3_BUCKET_NAME:-asksimba-images} + - S3_REGION=${S3_REGION:-garage} + - OLLAMA_HOST=${OLLAMA_HOST:-http://localhost:11434} + - FERNET_KEY=${FERNET_KEY} + - GOOGLE_CALENDAR_ENABLED=${GOOGLE_CALENDAR_ENABLED:-} + - GOOGLE_WORKSPACE_CLI_CREDENTIALS_FILE=${GOOGLE_WORKSPACE_CLI_CREDENTIALS_FILE:-/app/config/gws-credentials.json} + - MAILGUN_API_KEY=${MAILGUN_API_KEY} + - MAILGUN_DOMAIN=${MAILGUN_DOMAIN} + - MAILGUN_WEBHOOK_SIGNING_KEY=${MAILGUN_WEBHOOK_SIGNING_KEY} + - MAILGUN_SIGNATURE_VALIDATION=${MAILGUN_SIGNATURE_VALIDATION:-true} + - EMAIL_RATE_LIMIT_MAX=${EMAIL_RATE_LIMIT_MAX:-5} + - EMAIL_RATE_LIMIT_WINDOW=${EMAIL_RATE_LIMIT_WINDOW:-300} + - SENDBLUE_API_KEY=${SENDBLUE_API_KEY} + - SENDBLUE_API_SECRET=${SENDBLUE_API_SECRET} + - SENDBLUE_FROM_NUMBER=${SENDBLUE_FROM_NUMBER} + - SENDBLUE_WEBHOOK_SECRET=${SENDBLUE_WEBHOOK_SECRET} + - SENDBLUE_SIGNATURE_VALIDATION=${SENDBLUE_SIGNATURE_VALIDATION:-true} + - ALLOWED_IMESSAGE_NUMBERS=${ALLOWED_IMESSAGE_NUMBERS} + - SIMBAKIT_URL=${SIMBAKIT_URL:-http://simbakit:8585} + depends_on: + postgres: + condition: service_healthy + volumes: + - ./obvault:/app/data/obsidian + - ./credentials.json:/app/config/gws-credentials.json:ro + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/api/health"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 60s + restart: unless-stopped + services: postgres: image: pgvector/pgvector:pg16 @@ -19,76 +94,20 @@ services: retries: 5 restart: unless-stopped - raggr: - build: - context: . - dockerfile: Dockerfile - image: torrtle/simbarag:latest + # Blue-green deployment: exactly one of raggr-blue/raggr-green serves at a + # time; Caddy health-checks both ports and routes to the healthy one. + # Use scripts/deploy.sh (make redeploy) to swap colors with zero downtime. + raggr-blue: + <<: *raggr-common + profiles: ["blue"] ports: - "8080:8080" - environment: - - PAPERLESS_TOKEN=${PAPERLESS_TOKEN} - - BASE_URL=${BASE_URL} - - OLLAMA_URL=${OLLAMA_URL:-http://localhost:11434} - - OPENAI_API_KEY=${OPENAI_API_KEY} - - EMBEDDING_SERVER_URL=${EMBEDDING_SERVER_URL} - - EMBEDDING_MODEL_NAME=${EMBEDDING_MODEL_NAME} - - JWT_SECRET_KEY=${JWT_SECRET_KEY} - - LLAMA_SERVER_URL=${LLAMA_SERVER_URL} - - LLAMA_MODEL_NAME=${LLAMA_MODEL_NAME} - - OIDC_ISSUER=${OIDC_ISSUER} - - OIDC_CLIENT_ID=${OIDC_CLIENT_ID} - - OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET} - - OIDC_REDIRECT_URI=${OIDC_REDIRECT_URI} - - OIDC_USE_DISCOVERY=${OIDC_USE_DISCOVERY:-true} - - DATABASE_URL=${DATABASE_URL:-postgres://raggr:changeme@postgres:5432/raggr} - - TAVILY_API_KEY=${TAVILIY_API_KEY} - - YNAB_ACCESS_TOKEN=${YNAB_ACCESS_TOKEN} - - YNAB_BUDGET_ID=${YNAB_BUDGET_ID} - - MEALIE_BASE_URL=${MEALIE_BASE_URL} - - MEALIE_API_TOKEN=${MEALIE_API_TOKEN} - - TWILIO_ACCOUNT_SID=${TWILIO_ACCOUNT_SID} - - TWILIO_AUTH_TOKEN=${TWILIO_AUTH_TOKEN} - - TWILIO_WHATSAPP_NUMBER=${TWILIO_WHATSAPP_NUMBER} - - ALLOWED_WHATSAPP_NUMBERS=${ALLOWED_WHATSAPP_NUMBERS} - - TWILIO_SIGNATURE_VALIDATION=${TWILIO_SIGNATURE_VALIDATION:-true} - - TWILIO_WEBHOOK_URL=${TWILIO_WEBHOOK_URL:-} - - OBSIDIAN_EMAIL=${OBSIDIAN_EMAIL} - - OBSIDIAN_PASSWORD=${OBSIDIAN_PASSWORD} - - OBSIDIAN_VAULT_ID=${OBSIDIAN_VAULT_ID} - - OBSIDIAN_E2E_PASSWORD=${OBSIDIAN_E2E_PASSWORD} - - OBSIDIAN_DEVICE_NAME=${OBSIDIAN_DEVICE_NAME} - - OBSIDIAN_CONTINUOUS_SYNC=${OBSIDIAN_CONTINUOUS_SYNC:-false} - - OBSIDIAN_VAULT_PATH=${OBSIDIAN_VAULT_PATH:-/app/data/obsidian} - - S3_ENDPOINT_URL=${S3_ENDPOINT_URL} - - S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID} - - S3_SECRET_ACCESS_KEY=${S3_SECRET_ACCESS_KEY} - - S3_BUCKET_NAME=${S3_BUCKET_NAME:-asksimba-images} - - S3_REGION=${S3_REGION:-garage} - - OLLAMA_HOST=${OLLAMA_HOST:-http://localhost:11434} - - FERNET_KEY=${FERNET_KEY} - - GOOGLE_CALENDAR_ENABLED=${GOOGLE_CALENDAR_ENABLED:-} - - GOOGLE_WORKSPACE_CLI_CREDENTIALS_FILE=${GOOGLE_WORKSPACE_CLI_CREDENTIALS_FILE:-/app/config/gws-credentials.json} - - MAILGUN_API_KEY=${MAILGUN_API_KEY} - - MAILGUN_DOMAIN=${MAILGUN_DOMAIN} - - MAILGUN_WEBHOOK_SIGNING_KEY=${MAILGUN_WEBHOOK_SIGNING_KEY} - - MAILGUN_SIGNATURE_VALIDATION=${MAILGUN_SIGNATURE_VALIDATION:-true} - - EMAIL_RATE_LIMIT_MAX=${EMAIL_RATE_LIMIT_MAX:-5} - - EMAIL_RATE_LIMIT_WINDOW=${EMAIL_RATE_LIMIT_WINDOW:-300} - - SENDBLUE_API_KEY=${SENDBLUE_API_KEY} - - SENDBLUE_API_SECRET=${SENDBLUE_API_SECRET} - - SENDBLUE_FROM_NUMBER=${SENDBLUE_FROM_NUMBER} - - SENDBLUE_WEBHOOK_SECRET=${SENDBLUE_WEBHOOK_SECRET} - - SENDBLUE_SIGNATURE_VALIDATION=${SENDBLUE_SIGNATURE_VALIDATION:-true} - - ALLOWED_IMESSAGE_NUMBERS=${ALLOWED_IMESSAGE_NUMBERS} - - SIMBAKIT_URL=${SIMBAKIT_URL:-http://simbakit:8585} - depends_on: - postgres: - condition: service_healthy - volumes: - - ./obvault:/app/data/obsidian - - ./credentials.json:/app/config/gws-credentials.json:ro - restart: unless-stopped + + raggr-green: + <<: *raggr-common + profiles: ["green"] + ports: + - "8081:8080" simbakit: build: diff --git a/docs/deployment.md b/docs/deployment.md index 1f37b4f..46ea48a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -96,20 +96,88 @@ The dev environment: - Rebuilds frontend on changes - Auto-applies migrations on startup -### Production +### Production (zero-downtime blue-green) + +The app runs as one of two identical compose services: `raggr-blue` +(host port 8080) or `raggr-green` (host port 8081), gated behind compose +profiles so only one color runs at a time. Caddy health-checks both ports and +routes traffic to whichever is healthy. ```bash -# Build and deploy -docker compose build raggr -docker compose up -d +# Pull latest and swap colors with zero downtime +make redeploy -# View logs -docker compose logs -f raggr +# Or run the swap directly +./scripts/deploy.sh -# Verify migrations applied -docker compose exec raggr aerich history +# View logs / exec into the active color (Makefile resolves the color) +make logs +make migrate-history ``` +`scripts/deploy.sh` does the following: + +1. Detects the running color via `docker compose ps`. +2. Builds the inactive color. +3. Starts it with `docker compose up -d --wait`, which blocks until the + container healthcheck (`GET /api/health`) passes. +4. If the new color never becomes healthy, it is stopped and the old color + keeps serving (the script exits non-zero). +5. Otherwise the old color is stopped with a 30s grace period so in-flight + SSE streams can finish, then removed. + +To target a color manually, enable its profile: + +```bash +docker compose --profile blue up -d raggr-blue +docker compose --profile blue --profile green ps +``` + +#### Caddy configuration + +Caddy needs both upstreams with active health checks: + +```caddyfile +ask.torrtle.co { + reverse_proxy host.docker.internal:8080 host.docker.internal:8081 { + lb_policy first # prefer blue when both are healthy + lb_try_duration 5s # retry on the other upstream during the swap + lb_try_interval 250ms + health_uri /api/health + health_interval 2s + health_timeout 2s + fail_duration 10s + } +} +``` + +#### One-time cutover from the old `raggr` service + +The first deploy after this change lands needs manual steps, because the old +`raggr` container still holds port 8080 and the deploy script doesn't know +about it: + +```bash +# 1. Update the Caddy config to the two-upstream block below and reload Caddy +# 2. On the server: +git pull +docker compose --profile green up -d --build --wait raggr-green # port 8081, no conflict +docker stop -t 30 $(docker ps -q --filter name=raggr-1) # old raggr container +docker rm $(docker ps -aq --filter name=raggr-1) +# From here on, `make redeploy` handles swaps (green is now the active color) +``` + +#### Blue-green caveats + +- **Migrations must be backward-compatible.** The new color runs + `aerich upgrade` on startup while the old color is still serving the + previous code. Additive changes (new tables/columns) are safe; destructive + changes (drop/rename a column the old code reads) need a two-step deploy: + ship code that stops using the column first, then drop it in a later deploy. +- **Obsidian sync overlap.** During the ~30s swap window both containers run + pull-only `ob sync` against the shared `./obvault` mount. Both are + read-only pulls, so this is low risk, but avoid deploying mid-reindex. + ### Fresh Deploy (New Database) On first deploy with an empty database, `startup-dev.sh` runs `aerich init-db` instead of `aerich upgrade`. This creates all tables from the current models. diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..4849a49 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Zero-downtime blue-green deploy. +# +# Builds the inactive color, starts it, waits for its healthcheck to pass, +# then gracefully stops the old color. Caddy health-checks both ports +# (8080 = blue, 8081 = green) and routes traffic to whichever is healthy, +# so the swap is invisible to clients. +# +# Caveats: +# - Migrations run on the new container's startup while the old one still +# serves, so they must be backward-compatible (additive changes only; +# destructive changes need a two-step deploy). +# - During the brief overlap both containers run pull-only Obsidian sync +# against the shared ./obvault mount. +set -euo pipefail + +cd "$(dirname "$0")/.." + +running_services=$(docker compose ps --status running --services) + +if echo "$running_services" | grep -qx "raggr-blue"; then + OLD=blue NEW=green +elif echo "$running_services" | grep -qx "raggr-green"; then + OLD=green NEW=blue +else + OLD="" NEW=blue +fi + +echo "==> Active: ${OLD:-none} | Deploying: $NEW" + +echo "==> Building raggr-$NEW..." +docker compose --profile "$NEW" build "raggr-$NEW" + +echo "==> Starting raggr-$NEW and waiting for healthy..." +if ! docker compose --profile "$NEW" up -d --wait "raggr-$NEW"; then + echo "!! raggr-$NEW failed to become healthy${OLD:+; raggr-$OLD is still serving}" >&2 + docker compose --profile "$NEW" logs --tail 50 "raggr-$NEW" >&2 || true + docker compose --profile "$NEW" stop -t 5 "raggr-$NEW" || true + exit 1 +fi + +if [ -n "$OLD" ]; then + echo "==> raggr-$NEW healthy; stopping raggr-$OLD (30s grace for in-flight streams)..." + docker compose --profile "$OLD" stop -t 30 "raggr-$OLD" + docker compose --profile "$OLD" rm -f "raggr-$OLD" +fi + +echo "==> Deploy complete: raggr-$NEW is live"