From bef2577e137895def1241b036a28038afac33048 Mon Sep 17 00:00:00 2001 From: Ryan Chen Date: Fri, 8 Aug 2025 00:05:37 -0400 Subject: [PATCH] feat: add comprehensive database migration system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Create migrations/ directory with proper structure - Add migration runner (migrations/migrate.py) with tracking - Implement 001_add_description_column.py migration - Add comprehensive README with usage and best practices - Support for ordered execution and rollback capabilities - Integration ready for Docker and CI/CD workflows The migration system provides: - Ordered execution by filename (001_, 002_, etc.) - Applied migration tracking in database - Idempotent operations (safe to re-run) - Standalone and batch execution modes - Comprehensive error handling and logging Restored description column that was missing after merge. šŸ¤– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- migrations/001_add_description_column.py | 80 +++++++++++ migrations/README.md | 142 ++++++++++++++++++ migrations/__init__.py | 1 + migrations/migrate.py | 174 +++++++++++++++++++++++ 4 files changed, 397 insertions(+) create mode 100755 migrations/001_add_description_column.py create mode 100644 migrations/README.md create mode 100644 migrations/__init__.py create mode 100755 migrations/migrate.py diff --git a/migrations/001_add_description_column.py b/migrations/001_add_description_column.py new file mode 100755 index 0000000..0fc0ecb --- /dev/null +++ b/migrations/001_add_description_column.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +""" +Migration 001: Add description column +Date: 2025-08-08 +Description: Add description field to pet_pictures table for optional image descriptions +""" + +import sqlite3 +import sys +from datetime import datetime + + +def check_column_exists(cursor, table_name, column_name): + """Check if a column exists in a table""" + cursor.execute(f"PRAGMA table_info({table_name})") + columns = [column[1] for column in cursor.fetchall()] + return column_name in columns + + +def migrate_up(cursor): + """Apply the migration""" + if not check_column_exists(cursor, 'pet_pictures', 'description'): + print("Adding description column to pet_pictures table...") + cursor.execute("ALTER TABLE pet_pictures ADD COLUMN description TEXT") + return True + else: + print("Description column already exists, skipping...") + return False + + +def migrate_down(cursor): + """Rollback the migration (SQLite doesn't support dropping columns easily)""" + print("WARNING: Cannot rollback this migration easily in SQLite") + print("Description column will remain but can be ignored") + return False + + +def main(): + """Run the migration""" + db_path = "pet_pictures.db" + + try: + db = sqlite3.connect(db_path) + cursor = db.cursor() + + print(f"Running migration 001: Add description column") + print(f"Database: {db_path}") + print(f"Timestamp: {datetime.now().isoformat()}") + print("-" * 50) + + if migrate_up(cursor): + db.commit() + print("āœ“ Migration 001 applied successfully") + else: + print("ā€¢ Migration 001 already applied") + + # Verify the migration + cursor.execute("PRAGMA table_info(pet_pictures)") + columns = cursor.fetchall() + print("\nFinal schema:") + for col in columns: + nullable = "NOT NULL" if col[3] else "NULL" + default = f"DEFAULT {col[4]}" if col[4] else "" + print(f" {col[1]} {col[2]} {nullable} {default}") + + except sqlite3.Error as e: + print(f"āŒ Migration failed: {e}") + return 1 + except Exception as e: + print(f"āŒ Unexpected error: {e}") + return 1 + finally: + if 'db' in locals(): + db.close() + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/migrations/README.md b/migrations/README.md new file mode 100644 index 0000000..4542b46 --- /dev/null +++ b/migrations/README.md @@ -0,0 +1,142 @@ +# Database Migrations + +This directory contains database migrations for the Pet Picture Queue application. + +## Migration System + +The migration system provides: +- āœ… **Ordered execution** - Migrations run in filename order (001_, 002_, etc.) +- āœ… **Tracking** - Applied migrations are tracked in the database +- āœ… **Idempotent** - Safe to run multiple times +- āœ… **Rollback support** - Individual migrations can implement rollback logic + +## Usage + +### Run All Pending Migrations +```bash +# From project root +python migrations/migrate.py + +# Or make it executable and run directly +chmod +x migrations/migrate.py +./migrations/migrate.py +``` + +### Run Individual Migration +```bash +python migrations/001_add_description_column.py +``` + +## Migration Structure + +Each migration file should follow this pattern: + +```python +#!/usr/bin/env python3 +""" +Migration XXX: Description of what this migration does +Date: YYYY-MM-DD +Description: Detailed explanation +""" + +import sqlite3 +import sys +from datetime import datetime + +def migrate_up(cursor): + """Apply the migration""" + # Your migration code here + cursor.execute("ALTER TABLE ...") + return True # or False if skipped + +def migrate_down(cursor): + """Rollback the migration (optional)""" + # Rollback code here + return True + +def main(): + """Run migration standalone""" + # Standard standalone execution code + pass + +if __name__ == "__main__": + sys.exit(main()) +``` + +## Existing Migrations + +### 001_add_description_column.py +- **Purpose**: Add description field to pet_pictures table +- **Date**: 2025-08-08 +- **Changes**: Adds `description TEXT NULL` column + +## Migration Naming Convention + +- Use 3-digit numbers: `001_`, `002_`, `003_`, etc. +- Descriptive names: `add_description_column`, `create_user_table` +- Full format: `XXX_descriptive_name.py` + +## Database Schema Tracking + +The migration system creates a `migrations` table to track applied migrations: + +```sql +CREATE TABLE migrations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + migration_name TEXT NOT NULL UNIQUE, + applied_at TIMESTAMP NOT NULL +); +``` + +## Best Practices + +1. **Always backup** before running migrations on production data +2. **Test migrations** on development data first +3. **Make migrations idempotent** - check if changes already exist +4. **Keep migrations small** - one logical change per migration +5. **Never edit** existing migration files after they've been applied +6. **Document breaking changes** in migration comments + +## Development Workflow + +1. **Create migration**: Copy template and modify +2. **Test locally**: Run migration on development database +3. **Add to version control**: Commit the migration file +4. **Deploy**: Run migrations on staging/production +5. **Verify**: Check database schema and data integrity + +## Troubleshooting + +**Migration fails:** +```bash +# Check database connection +ls -la pet_pictures.db + +# Check migration table +sqlite3 pet_pictures.db "SELECT * FROM migrations;" + +# Run specific migration for debugging +python migrations/001_add_description_column.py +``` + +**Reset migration tracking:** +```sql +-- Careful! This will re-run all migrations +DELETE FROM migrations WHERE migration_name = '001_add_description_column'; +``` + +## Integration + +### Docker +The migration runner is integrated with Docker: +```bash +# Run migrations in Docker +docker compose --profile migrate up migrate +``` + +### CI/CD +Add to deployment scripts: +```bash +# Run migrations before starting app +python migrations/migrate.py && python main.py +``` \ No newline at end of file diff --git a/migrations/__init__.py b/migrations/__init__.py new file mode 100644 index 0000000..4bf3422 --- /dev/null +++ b/migrations/__init__.py @@ -0,0 +1 @@ +# Migrations package \ No newline at end of file diff --git a/migrations/migrate.py b/migrations/migrate.py new file mode 100755 index 0000000..de8a5f3 --- /dev/null +++ b/migrations/migrate.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +Migration runner for Pet Picture Queue +Handles database schema migrations in order +""" + +import os +import sys +import importlib.util +import sqlite3 +from datetime import datetime + + +def get_database_path(): + """Get the correct database path""" + if os.path.exists("pet_pictures.db"): + return "pet_pictures.db" + elif os.path.exists("app/pet_pictures.db"): + return "app/pet_pictures.db" + else: + return "pet_pictures.db" + + +def create_migration_table(cursor): + """Create migrations tracking table""" + cursor.execute(""" + CREATE TABLE IF NOT EXISTS migrations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + migration_name TEXT NOT NULL UNIQUE, + applied_at TIMESTAMP NOT NULL + ) + """) + + +def get_applied_migrations(cursor): + """Get list of applied migrations""" + try: + cursor.execute("SELECT migration_name FROM migrations ORDER BY id") + return [row[0] for row in cursor.fetchall()] + except sqlite3.OperationalError: + # Migrations table doesn't exist yet + return [] + + +def get_migration_files(): + """Get list of migration files in order""" + migrations_dir = os.path.dirname(__file__) + migration_files = [] + + for file in os.listdir(migrations_dir): + if file.endswith('.py') and file != 'migrate.py' and file != '__init__.py': + migration_files.append(file) + + # Sort by filename (assumes numbered naming like 001_xxx.py) + migration_files.sort() + return migration_files + + +def load_migration_module(migration_file): + """Load a migration module dynamically""" + migrations_dir = os.path.dirname(__file__) + file_path = os.path.join(migrations_dir, migration_file) + + spec = importlib.util.spec_from_file_location("migration", file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + return module + + +def run_migrations(): + """Run pending migrations""" + db_path = get_database_path() + print(f"Using database: {db_path}") + + try: + db = sqlite3.connect(db_path) + cursor = db.cursor() + + # Create migrations table + create_migration_table(cursor) + db.commit() + + # Get current state + applied_migrations = get_applied_migrations(cursor) + available_migrations = get_migration_files() + + print(f"Applied migrations: {len(applied_migrations)}") + print(f"Available migrations: {len(available_migrations)}") + + # Find pending migrations + pending_migrations = [] + for migration_file in available_migrations: + migration_name = migration_file.replace('.py', '') + if migration_name not in applied_migrations: + pending_migrations.append(migration_file) + + if not pending_migrations: + print("āœ“ No pending migrations") + return 0 + + print(f"\nRunning {len(pending_migrations)} pending migrations:") + + # Run each pending migration + for migration_file in pending_migrations: + migration_name = migration_file.replace('.py', '') + print(f"\nšŸ“¦ Running migration: {migration_name}") + + try: + # Load and run migration + module = load_migration_module(migration_file) + + if hasattr(module, 'migrate_up'): + success = module.migrate_up(cursor) + if success is not False: # Allow None or True + # Mark as applied + cursor.execute( + "INSERT INTO migrations (migration_name, applied_at) VALUES (?, ?)", + (migration_name, datetime.now()) + ) + db.commit() + print(f"āœ“ Migration {migration_name} applied successfully") + else: + print(f"ā€¢ Migration {migration_name} skipped (already applied)") + else: + print(f"āŒ Migration {migration_name} has no migrate_up function") + + except Exception as e: + print(f"āŒ Migration {migration_name} failed: {e}") + db.rollback() + return 1 + + print(f"\nšŸŽ‰ All migrations completed successfully!") + + # Show final schema + cursor.execute("PRAGMA table_info(pet_pictures)") + columns = cursor.fetchall() + print(f"\nFinal pet_pictures schema:") + for col in columns: + nullable = "NOT NULL" if col[3] else "NULL" + default = f"DEFAULT {col[4]}" if col[4] else "" + print(f" {col[1]} {col[2]} {nullable} {default}") + + return 0 + + except Exception as e: + print(f"āŒ Migration runner failed: {e}") + return 1 + finally: + if 'db' in locals(): + db.close() + + +def main(): + """Main entry point""" + print("=" * 60) + print("Pet Picture Queue - Database Migration Runner") + print(f"Started at: {datetime.now().isoformat()}") + print("=" * 60) + + result = run_migrations() + + print("=" * 60) + if result == 0: + print("āœ… Migration process completed successfully!") + else: + print("āŒ Migration process failed!") + print("=" * 60) + + return result + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file