From 1fd2e860b23d7662fe139fd36e05d139dd7132f2 Mon Sep 17 00:00:00 2001 From: Ryan Chen Date: Sat, 31 Jan 2026 16:47:57 -0500 Subject: [PATCH] nani --- docker-compose.dev.yml | 1 + docs/deployment.md | 187 ++++++++++++++++++ .../models/0_20251225052005_init.py | 71 ------- .../1_20260131000000_add_ldap_groups.py | 15 -- .../models/1_20260131214411_None.py | 72 +++++++ services/raggr/oidc_config.py | 1 + services/raggr/scripts/user_message_stats.py | 79 ++++++++ 7 files changed, 340 insertions(+), 86 deletions(-) create mode 100644 docs/deployment.md delete mode 100644 services/raggr/migrations/models/0_20251225052005_init.py delete mode 100644 services/raggr/migrations/models/1_20260131000000_add_ldap_groups.py create mode 100644 services/raggr/migrations/models/1_20260131214411_None.py create mode 100644 services/raggr/scripts/user_message_stats.py diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2e60619..9227d7a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -46,6 +46,7 @@ services: condition: service_healthy volumes: - chromadb_data:/app/data/chromadb + - ./services/raggr/migrations:/app/migrations # Bind mount for migrations (bidirectional) develop: watch: # Sync+restart on any file change under services/raggr diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..620075f --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,187 @@ +# Deployment & Migrations Guide + +This document covers database migrations and deployment workflows for SimbaRAG. + +## Migration Workflow + +Migrations are managed by [Aerich](https://github.com/tortoise/aerich), the migration tool for Tortoise ORM. + +### Key Principles + +1. **Generate migrations in Docker** - Aerich needs database access to detect schema changes +2. **Migrations auto-apply on startup** - Both `startup.sh` and `startup-dev.sh` run `aerich upgrade` +3. **Commit migrations to git** - Migration files must be in the repo for production deploys + +### Generating a New Migration + +#### Development (Recommended) + +With `docker-compose.dev.yml`, your local `services/raggr` directory is synced to the container. Migrations generated inside the container appear on your host automatically. + +```bash +# 1. Start the dev environment +docker compose -f docker-compose.dev.yml up -d + +# 2. Generate migration (runs inside container, syncs to host) +docker compose -f docker-compose.dev.yml exec raggr aerich migrate --name describe_your_change + +# 3. Verify migration was created +ls services/raggr/migrations/models/ + +# 4. Commit the migration +git add services/raggr/migrations/ +git commit -m "Add migration: describe_your_change" +``` + +#### Production Container + +For production, migration files are baked into the image. You must generate migrations in dev first. + +```bash +# If you need to generate a migration from production (not recommended): +docker compose exec raggr aerich migrate --name describe_your_change + +# Copy the file out of the container +docker cp $(docker compose ps -q raggr):/app/migrations/models/ ./services/raggr/migrations/ +``` + +### Applying Migrations + +Migrations apply automatically on container start via the startup scripts. + +**Manual application (if needed):** + +```bash +# Dev +docker compose -f docker-compose.dev.yml exec raggr aerich upgrade + +# Production +docker compose exec raggr aerich upgrade +``` + +### Checking Migration Status + +```bash +# View applied migrations +docker compose exec raggr aerich history + +# View pending migrations +docker compose exec raggr aerich heads +``` + +### Rolling Back + +```bash +# Downgrade one migration +docker compose exec raggr aerich downgrade + +# Downgrade to specific version +docker compose exec raggr aerich downgrade -v 1 +``` + +## Deployment Workflows + +### Development + +```bash +# Start with watch mode (auto-restarts on file changes) +docker compose -f docker-compose.dev.yml up + +# Or with docker compose watch (requires Docker Compose v2.22+) +docker compose -f docker-compose.dev.yml watch +``` + +The dev environment: +- Syncs `services/raggr/` to `/app` in the container +- Rebuilds frontend on changes +- Auto-applies migrations on startup + +### Production + +```bash +# Build and deploy +docker compose build raggr +docker compose up -d + +# View logs +docker compose logs -f raggr + +# Verify migrations applied +docker compose exec raggr aerich history +``` + +### Fresh Deploy (New Database) + +On first deploy with an empty database, `startup-dev.sh` runs `aerich init-db` instead of `aerich upgrade`. This creates all tables from the current models. + +For production (`startup.sh`), ensure the database exists and run: + +```bash +# If aerich table doesn't exist yet +docker compose exec raggr aerich init-db + +# Or if migrating from existing schema +docker compose exec raggr aerich upgrade +``` + +## Troubleshooting + +### "No migrations found" on startup + +The `migrations/models/` directory is empty or not copied into the image. + +**Fix:** Ensure migrations are committed and the Dockerfile copies them: +```dockerfile +COPY migrations ./migrations +``` + +### Migration fails with "relation already exists" + +The database has tables but aerich doesn't know about them (fresh aerich setup on existing DB). + +**Fix:** Fake the initial migration: +```bash +# Mark initial migration as applied without running it +docker compose exec raggr aerich upgrade --fake +``` + +### Model changes not detected + +Aerich compares models against the last migration's state. If state is out of sync: + +```bash +# Regenerate migration state (dangerous - review carefully) +docker compose exec raggr aerich migrate --name fix_state +``` + +### Database connection errors + +Ensure PostgreSQL is healthy before running migrations: + +```bash +# Check postgres status +docker compose ps postgres + +# Wait for postgres then run migrations +docker compose exec raggr bash -c "sleep 5 && aerich upgrade" +``` + +## File Reference + +| File | Purpose | +|------|---------| +| `services/raggr/pyproject.toml` | Aerich config (`[tool.aerich]` section) | +| `services/raggr/migrations/models/` | Migration files | +| `services/raggr/startup.sh` | Production startup (runs `aerich upgrade`) | +| `services/raggr/startup-dev.sh` | Dev startup (runs `aerich upgrade` or `init-db`) | +| `services/raggr/app.py` | Contains `TORTOISE_CONFIG` | + +## Quick Reference + +| Task | Command | +|------|---------| +| Generate migration | `docker compose -f docker-compose.dev.yml exec raggr aerich migrate --name name` | +| Apply migrations | `docker compose exec raggr aerich upgrade` | +| View history | `docker compose exec raggr aerich history` | +| Rollback | `docker compose exec raggr aerich downgrade` | +| Fresh init | `docker compose exec raggr aerich init-db` | diff --git a/services/raggr/migrations/models/0_20251225052005_init.py b/services/raggr/migrations/models/0_20251225052005_init.py deleted file mode 100644 index 9a8e557..0000000 --- a/services/raggr/migrations/models/0_20251225052005_init.py +++ /dev/null @@ -1,71 +0,0 @@ -from tortoise import BaseDBAsyncClient - -RUN_IN_TRANSACTION = True - - -async def upgrade(db: BaseDBAsyncClient) -> str: - return """ - CREATE TABLE IF NOT EXISTS "users" ( - "id" UUID NOT NULL PRIMARY KEY, - "username" VARCHAR(255) NOT NULL, - "password" BYTEA, - "email" VARCHAR(100) NOT NULL UNIQUE, - "oidc_subject" VARCHAR(255) UNIQUE, - "auth_provider" VARCHAR(50) NOT NULL DEFAULT 'local', - "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP -); -CREATE INDEX IF NOT EXISTS "idx_users_oidc_su_5aec5a" ON "users" ("oidc_subject"); -CREATE TABLE IF NOT EXISTS "conversations" ( - "id" UUID NOT NULL PRIMARY KEY, - "name" VARCHAR(255) NOT NULL, - "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - "user_id" UUID REFERENCES "users" ("id") ON DELETE CASCADE -); -CREATE TABLE IF NOT EXISTS "conversation_messages" ( - "id" UUID NOT NULL PRIMARY KEY, - "text" TEXT NOT NULL, - "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - "speaker" VARCHAR(10) NOT NULL, - "conversation_id" UUID NOT NULL REFERENCES "conversations" ("id") ON DELETE CASCADE -); -COMMENT ON COLUMN "conversation_messages"."speaker" IS 'USER: user\nSIMBA: simba'; -CREATE TABLE IF NOT EXISTS "aerich" ( - "id" SERIAL NOT NULL PRIMARY KEY, - "version" VARCHAR(255) NOT NULL, - "app" VARCHAR(100) NOT NULL, - "content" JSONB NOT NULL -);""" - - -async def downgrade(db: BaseDBAsyncClient) -> str: - return """ - """ - - -MODELS_STATE = ( - "eJztmmtP4zgUhv9KlE+MxCLoUGaEViulpex0Z9qO2nR3LjuK3MRtvSROJnYGKsR/X9u5J0" - "56AUqL+gXosU9sPz7OeY/Lveq4FrTJSdvFv6BPAEUuVi+VexUDB7I/pO3Higo8L23lBgom" - "tnAwMz1FC5gQ6gOTssYpsAlkJgsS00deNBgObJsbXZN1RHiWmgKMfgbQoO4M0jn0WcP3H8" - "yMsAXvIIk/ejfGFEHbys0bWXxsYTfowhO28bh7dS168uEmhunagYPT3t6Czl2cdA8CZJ1w" - "H942gxj6gEIrsww+y2jZsSmcMTNQP4DJVK3UYMEpCGwOQ/19GmCTM1DESPzH+R/qGngYao" - "4WYcpZ3D+Eq0rXLKwqH6r9QRsevb14I1bpEjrzRaMgoj4IR0BB6Cq4piDF7xLK9hz4cpRx" - "/wJMNtFNMMaGlGMaQzHIGNBm1FQH3Bk2xDM6Zx8bzWYNxr+1oSDJegmULovrMOr7UVMjbO" - "NIU4SmD/mSDUDLIK9YC0UOlMPMexaQWpHrSfzHjgJma7AG2F5Eh6CGr97tdUa61vvMV+IQ" - "8tMWiDS9w1sawrooWI8uCluRPET5p6t/UPhH5dug3ynGftJP/6byOYGAugZ2bw1gZc5rbI" - "3B5DY28KwNNzbvedjYF93YaPKZfSXQN9bLIBmXR6SRaG5b3MTNkwZPvdMbac7gMMrwrl0f" - "ohn+CBcCYZfNA2BTliwi0TGOHrOr0FJrOgsf3CZqJBsUbHVsTZCG2VMbtbWrjioYToB5cw" - "t8y6iA6UBCwAySMtBW5Hn9cQjtRJrJWWYFXC984m6+VarYClZuw80wytErNzkNp2gBmK3b" - "isbmI9XQWaKCMxBXE8NGdiMPonivRTGFd5KUrzOrHGXcf19EcV0q73zRc1k8lr5HPe3Lm1" - "wm/zTo/xl3z0jl9qdB66CQX6OQKitk4kFwIxMDvIDs4MApSYHc7mbcX/joqONRZ3ip8Iz+" - "Lx51ey3tUiHImQB1tS3OVZlnpysUmWenlTUmbyocoGyiWe81L3F9ynf+nkpYs3Dh9UgpW7" - "w/21mKSzWtJFzW1bbPqeREzSCRbnEtUa3V+NE+aLP912Z8H9e9tMz67ItG28LFpQcIuXV9" - "SWS2EAb+Qg4z61WAOVnQsP7Z1ZJeBq/F9WpWbjFkrW5fG36VS964fzZuW1/1jlagCx2A7H" - "WiNHF4mhBdfuKfMkDPTlcTPXWqpyR7XGSZBgkm/0FTUjlUkyz6bQS0GKTb5fksB55p+bnh" - "+e4vZFWJdjnQkuP23qKq7ZrAfkQaynNtrhKmzeoobZa1+aG4fZ3F7eHrn1exscntcqlIWX" - "Y1X/pfh6e5n99lgbTde3kN+sicq5J6Lmo5rqvoQNpnZ0q6Lq64IpZWdBxzIRiinX9RYSe+" - "HfmtcXb+7vz924vz96yLmElieVfzMuj29SUVHD8I0muXav2RcTnUb6mcY0djHREXdt9PgM" - "9SX7ARKcSS9P7XaNCvvE6NXQogx5gt8LuFTHqs2IjQH7uJtYYiX3X9dz/Fr3kKuZk/oCW7" - "eN3mZeHD/9BpOYI=" -) diff --git a/services/raggr/migrations/models/1_20260131000000_add_ldap_groups.py b/services/raggr/migrations/models/1_20260131000000_add_ldap_groups.py deleted file mode 100644 index d385895..0000000 --- a/services/raggr/migrations/models/1_20260131000000_add_ldap_groups.py +++ /dev/null @@ -1,15 +0,0 @@ -from tortoise import BaseDBAsyncClient - -RUN_IN_TRANSACTION = True - - -async def upgrade(db: BaseDBAsyncClient) -> str: - return """ - ALTER TABLE "users" ADD COLUMN "ldap_groups" JSONB DEFAULT '[]'; - """ - - -async def downgrade(db: BaseDBAsyncClient) -> str: - return """ - ALTER TABLE "users" DROP COLUMN "ldap_groups"; - """ diff --git a/services/raggr/migrations/models/1_20260131214411_None.py b/services/raggr/migrations/models/1_20260131214411_None.py new file mode 100644 index 0000000..9a7f911 --- /dev/null +++ b/services/raggr/migrations/models/1_20260131214411_None.py @@ -0,0 +1,72 @@ +from tortoise import BaseDBAsyncClient + +RUN_IN_TRANSACTION = True + + +async def upgrade(db: BaseDBAsyncClient) -> str: + return """ + CREATE TABLE IF NOT EXISTS "users" ( + "id" UUID NOT NULL PRIMARY KEY, + "username" VARCHAR(255) NOT NULL, + "password" BYTEA, + "email" VARCHAR(100) NOT NULL UNIQUE, + "oidc_subject" VARCHAR(255) UNIQUE, + "auth_provider" VARCHAR(50) NOT NULL DEFAULT 'local', + "ldap_groups" JSONB NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS "idx_users_oidc_su_5aec5a" ON "users" ("oidc_subject"); +CREATE TABLE IF NOT EXISTS "conversations" ( + "id" UUID NOT NULL PRIMARY KEY, + "name" VARCHAR(255) NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + "user_id" UUID REFERENCES "users" ("id") ON DELETE CASCADE +); +CREATE TABLE IF NOT EXISTS "conversation_messages" ( + "id" UUID NOT NULL PRIMARY KEY, + "text" TEXT NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + "speaker" VARCHAR(10) NOT NULL, + "conversation_id" UUID NOT NULL REFERENCES "conversations" ("id") ON DELETE CASCADE +); +COMMENT ON COLUMN "conversation_messages"."speaker" IS 'USER: user\nSIMBA: simba'; +CREATE TABLE IF NOT EXISTS "aerich" ( + "id" SERIAL NOT NULL PRIMARY KEY, + "version" VARCHAR(255) NOT NULL, + "app" VARCHAR(100) NOT NULL, + "content" JSONB NOT NULL +);""" + + +async def downgrade(db: BaseDBAsyncClient) -> str: + return """ + """ + + +MODELS_STATE = ( + "eJztmm1v4jgQx78Kyquu1KtatnRX1emkQOkttwuceNinXhWZ2ICviZ1NnG1R1e9+tkmIkz" + "gUKFDY401bxh5s/zzO/Mfpo+FSiJzgpEbJT+QHgGFKjMvSo0GAi/gf2vbjkgE8L2kVBgYG" + "jnSwlZ6yBQwC5gOb8cYhcALETRAFto+9aDASOo4wUpt3xGSUmEKCf4TIYnSE2Bj5vOHmlp" + "sxgegBBfFH784aYuTA1LwxFGNLu8UmnrT1+42ra9lTDDewbOqELkl6exM2pmTWPQwxPBE+" + "om2ECPIBQ1BZhphltOzYNJ0xNzA/RLOpwsQA0RCEjoBh/D4MiS0YlORI4sf5H8YSeDhqgR" + "YTJlg8Pk1XlaxZWg0xVO2D2Tl6e/FGrpIGbOTLRknEeJKOgIGpq+SagJS/cyhrY+DrUcb9" + "MzD5RFfBGBsSjkkMxSBjQKtRM1zwYDmIjNiYfyxXKnMwfjY7kiTvJVFSHtfTqG9FTeVpm0" + "CaILR9JJZsAZYHecVbGHaRHmbaM4MURq4n8R87CpivAbaJM4kOwRy+vUaz3u2Zzb/FStwg" + "+OFIRGavLlrK0jrJWI8uMlsx+5LSl0bvQ0l8LH1vt+rZ2J/16303xJxAyKhF6L0FoHJeY2" + "sMJrWxoQdX3Ni052FjX3Vjo8kr+xog31ougyguL0gj0dy2uImrJw2Reod32pwhYOThXVMf" + "4RH5iCYSYYPPAxBblywi0dGPvmZXoSXWZBY+uJ+pETUo+Or4mhCbZk+zWzOv6oZkOAD23T" + "3woVUA00VBAEYoyAOtRp7XHzvImUkzPUtVwDWn37ibT5UitpIVLVOFUYpevsktu1kLIHzd" + "MBpbjDSHzjMqWIG4mBi21I08iOK9FsUMPWhSfo9b9Sjj/vsiiuel8vrXXiqLx9L3qGl+fZ" + "PK5J/arT/j7opUrn1qVw8K+VcUUnmFHHgI3OnEgCgg6yR0c1IgtbuK+ysfHaPfrXcuSyKj" + "/0O6jWbVvCwF2B0AY7EtTlWZZ6cLFJlnp4U1pmjKHCA10Sz3mNe4rvOZv6cS1s5ceL1Qym" + "bvz3aW4rOaVhMuy2rbTSo5WTNopFtcSxRrNXG0D9ps/7WZ2MdlLy1Vn33RaFu4uPRAENxT" + "XxOZVUyAP9HDVL0yMAcTNq1/drWk18GrCr2qyi2OrNpomZ1veskb91fjtvqtVzczdJELsL" + "NMlM4c1hOiz5/4dQbo2eliomee6snJHoqhbQXh4F9kayqHYpJZv5WAZoN0uzw3cuC5lh9b" + "nk9/Ylgk2vVAc47be4oaDrWB84I0lOZaWSRMK8VRWskFqQOBZ418GnqaO7y/uu2WHmnGLQ" + "O0T/gqbyC22XHJwQG73Rjem9vNpHix8vkXCdk7g8wzVXzB4SLhf3KRcHjV9kts7OwmP1cQ" + "PvcaJPd/Jet5F7LLYnS770BM5GN7bGhq56jleF71DJI+O1M+N0jBdby2ehaYM8EQ7fyrim" + "j5Juq38tn5u/P3by/O3/MuciYzy7s5D4NGq/dMtSwOgvaKq1jrKS6HWjmRzvxoLCOYp933" + "E+BGajk+IkNEk96LJbLi8lryeGO3DmuTx0tk2/Wnl6f/AHvgrXs=" +) diff --git a/services/raggr/oidc_config.py b/services/raggr/oidc_config.py index 76b58b2..fe3b996 100644 --- a/services/raggr/oidc_config.py +++ b/services/raggr/oidc_config.py @@ -1,6 +1,7 @@ """ OIDC Configuration for Authelia Integration """ + import os from typing import Dict, Any from authlib.jose import jwt diff --git a/services/raggr/scripts/user_message_stats.py b/services/raggr/scripts/user_message_stats.py new file mode 100644 index 0000000..0bb7820 --- /dev/null +++ b/services/raggr/scripts/user_message_stats.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +Script to show how many messages each user has written +""" + +import asyncio +from tortoise import Tortoise +from blueprints.users.models import User +from blueprints.conversation.models import Speaker +import os + + +async def get_user_message_stats(): + """Get message count statistics per user""" + + # Initialize database connection + database_url = os.getenv("DATABASE_URL", "sqlite://raggr.db") + await Tortoise.init( + db_url=database_url, + modules={ + "models": [ + "blueprints.users.models", + "blueprints.conversation.models", + ] + }, + ) + + print("\nšŸ“Š User Message Statistics\n") + print( + f"{'Username':<20} {'Total Messages':<15} {'User Messages':<15} {'Conversations':<15}" + ) + print("=" * 70) + + # Get all users + users = await User.all() + + total_users = 0 + total_messages = 0 + + for user in users: + # Get all conversations for this user + conversations = await user.conversations.all() + + if not conversations: + continue + + total_users += 1 + + # Count messages across all conversations + user_message_count = 0 + total_message_count = 0 + + for conversation in conversations: + messages = await conversation.messages.all() + total_message_count += len(messages) + + # Count only user messages (not assistant responses) + user_messages = [msg for msg in messages if msg.speaker == Speaker.USER] + user_message_count += len(user_messages) + + total_messages += user_message_count + + print( + f"{user.username:<20} {total_message_count:<15} {user_message_count:<15} {len(conversations):<15}" + ) + + print("=" * 70) + print("\nšŸ“ˆ Summary:") + print(f" Total active users: {total_users}") + print(f" Total user messages: {total_messages}") + print( + f" Average messages per user: {total_messages / total_users if total_users > 0 else 0:.1f}\n" + ) + + await Tortoise.close_connections() + + +if __name__ == "__main__": + asyncio.run(get_user_message_stats())