Checklist: 15 steps, CI/CD automates 10, human verifies 5
Result: 0 missed steps, 0 post-deploy incidents

# docker-compose.prod.yml
services:
  app:
    image: ghcr.io/bob/shopmetrics:latest
    restart: unless-stopped
    deploy:
      replicas: 2
      resources:
        limits:
          memory: 512M
          cpus: '1.0'
    env_file:
      - .env.production
    depends_on:
      mysql:
        condition: service_healthy
      redis:
        condition: service_healthy
    volumes:
      - storage:/var/www/html/storage
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:8080/health"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 10s
    networks:
      - internal

  queue-high:
    image: ghcr.io/bob/shopmetrics:latest
    restart: unless-stopped
    command: php artisan queue:work --queue=high --sleep=3 --tries=3 --max-time=3600
    deploy:
      replicas: 2
      resources:
        limits:
          memory: 256M
    env_file:
      - .env.production
    depends_on:
      mysql:
        condition: service_healthy
      redis:
        condition: service_healthy
    volumes:
      - storage:/var/www/html/storage
    networks:
      - internal

  queue-default:
    image: ghcr.io/bob/shopmetrics:latest
    restart: unless-stopped
    command: php artisan queue:work --queue=default --sleep=3 --tries=3 --max-time=3600
    deploy:
      replicas: 3
      resources:
        limits:
          memory: 256M
    env_file:
      - .env.production
    depends_on:
      mysql:
        condition: service_healthy
      redis:
        condition: service_healthy
    volumes:
      - storage:/var/www/html/storage
    networks:
      - internal

  scheduler:
    image: ghcr.io/bob/shopmetrics:latest
    restart: unless-stopped
    command: php artisan schedule:work --verbose
    env_file:
      - .env.production
    depends_on:
      mysql:
        condition: service_healthy
      redis:
        condition: service_healthy
    networks:
      - internal

  mysql:
    image: mysql:8.0
    restart: unless-stopped
    environment:
      MYSQL_DATABASE: ${DB_DATABASE}
      MYSQL_USER: ${DB_USERNAME}
      MYSQL_PASSWORD: ${DB_PASSWORD}
      MYSQL_ROOT_PASSWORD: ${DB_ROOT_PASSWORD}
    volumes:
      - mysql_data:/var/lib/mysql
      - ./docker/mysql/my.cnf:/etc/mysql/conf.d/my.cnf:ro
    healthcheck:
      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-p${DB_ROOT_PASSWORD}"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - internal

  redis:
    image: redis:7-alpine
    restart: unless-stopped
    command: >
      redis-server
      --requirepass ${REDIS_PASSWORD}
      --maxmemory 512mb
      --maxmemory-policy allkeys-lru
      --appendonly yes
    volumes:
      - redis_data:/data
    healthcheck:
      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - internal

  nginx:
    image: nginx:alpine
    restart: unless-stopped
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - ./docker/nginx/production.conf:/etc/nginx/conf.d/default.conf:ro
      - ./docker/nginx/ssl:/etc/nginx/ssl:ro
    depends_on:
      app:
        condition: service_healthy
    networks:
      - internal

volumes:
  mysql_data:
  redis_data:
  storage:

networks:
  internal:
    driver: bridge

# .env.production
APP_NAME=ShopMetrics
APP_ENV=production
APP_KEY=base64:xxx
APP_DEBUG=false
APP_URL=https://shopmetrics.io
APP_VERSION=1.0.0

DB_CONNECTION=mysql
DB_HOST=mysql
DB_PORT=3306
DB_DATABASE=shopmetrics
DB_USERNAME=shopmetrics
DB_PASSWORD=${DB_PASSWORD}
DB_ROOT_PASSWORD=${DB_ROOT_PASSWORD}

CACHE_DRIVER=redis
QUEUE_CONNECTION=redis
SESSION_DRIVER=redis
REDIS_HOST=redis
REDIS_PASSWORD=${REDIS_PASSWORD}

FILESYSTEM_DISK=s3
AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
AWS_DEFAULT_REGION=us-east-1
AWS_BUCKET=shopmetrics-reports

STRIPE_KEY=pk_live_xxx
STRIPE_SECRET=sk_live_xxx
STRIPE_WEBHOOK_SECRET=whsec_xxx

SENTRY_LARAVEL_DSN=https://xxx@sentry.io/xxx
SENTRY_TRACES_SAMPLE_RATE=0.2

LOG_CHANNEL=daily
LOG_LEVEL=warning

# Command executed successfully

# .github/workflows/deploy-production.yml
name: Deploy to Production

on:
  push:
    branches: [main]
  workflow_dispatch:  # Manual trigger option

concurrency: production  # Only one deploy at a time

env:
  REGISTRY: ghcr.io
  IMAGE: ${{ github.repository }}

jobs:
  test:
    runs-on: ubuntu-latest
    services:
      mysql:
        image: mysql:8.0
        env:
          MYSQL_DATABASE: shopmetrics_test
          MYSQL_USER: test
          MYSQL_PASSWORD: test
          MYSQL_ROOT_PASSWORD: test
        ports: ['3306:3306']
      redis:
        image: redis:7-alpine
        ports: ['6379:6379']
    steps:
      - uses: actions/checkout@v4
      - uses: shivammathur/setup-php@v2
        with:
          php-version: '8.3'
          coverage: xdebug
      - name: Install & Test
        run: |
          composer install --no-progress
          cp .env.example .env
          php artisan key:generate
          php artisan test --parallel --coverage-text=coverage.txt
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: coverage
          path: coverage.txt

  security:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: composer install --no-dev
      - name: Security audit
        run: composer audit

  build:
    needs: [test, security]
    runs-on: ubuntu-latest
    permissions:
      packages: write
    steps:
      - uses: actions/checkout@v4
      - uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: |
            ${{ env.REGISTRY }}/${{ env.IMAGE }}:latest
            ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ github.sha }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  deploy:
    needs: build
    runs-on: ubuntu-latest
    environment: production
    steps:
      - name: Deploy via SSH
        uses: appleboy/ssh-action@v1
        with:
          host: ${{ secrets.PROD_HOST }}
          username: deploy
          key: ${{ secrets.PROD_SSH_KEY }}
          script: |
            cd /opt/shopmetrics
            export IMAGE_TAG=${{ github.sha }}

            # Pull latest image
            docker compose -f docker-compose.prod.yml pull app queue-high queue-default scheduler

            # Rolling update: start new containers
            docker compose -f docker-compose.prod.yml up -d --no-deps --scale app=3 app
            sleep 10

            # Health check new containers
            for i in $(seq 1 5); do
              if curl -sf http://localhost:8080/health > /dev/null 2>&1; then
                echo "✓ Health check passed"
                break
              fi
              echo "Waiting for health check... ($i/5)"
              sleep 5
            done

            # Run migrations
            docker compose -f docker-compose.prod.yml exec -T app php artisan migrate --force

            # Restart workers (graceful)
            docker compose -f docker-compose.prod.yml exec -T app php artisan queue:restart

            # Scale down old containers
            docker compose -f docker-compose.prod.yml up -d --scale app=2

            # Clear and rebuild caches
            docker compose -f docker-compose.prod.yml exec -T app php artisan optimize:clear
            docker compose -f docker-compose.prod.yml exec -T app php artisan optimize

            echo "✓ Deploy complete: $IMAGE_TAG"

  verify:
    needs: deploy
    runs-on: ubuntu-latest
    steps:
      - name: Smoke test production
        run: |
          STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://shopmetrics.io/health)
          if [ "$STATUS" != "200" ]; then
            echo "✗ Health check failed: HTTP $STATUS"
            exit 1
          fi
          echo "✓ Production health check passed"

      - name: Notify Slack on success
        if: success()
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {"text":"✅ ShopMetrics deployed successfully: ${{ github.sha }}"}

      - name: Notify Slack on failure
        if: failure()
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {"text":"🚨 ShopMetrics deployment FAILED: ${{ github.sha }}"}

// Safe migration patterns for production

// ✅ SAFE: Add column (non-breaking)
Schema::table('orders', function (Blueprint $table) {
    $table->string('shipping_method')->nullable()->after('status');
    $table->index('shipping_method');
});

// ✅ SAFE: Add index (concurrent in PostgreSQL)
// MySQL: Use ALGORITHM=INPLACE for non-blocking index creation
DB::statement('ALTER TABLE orders ADD INDEX idx_orders_shipping (shipping_method) ALGORITHM=INPLACE');

// ⚠️ CAUTION: Remove column (two-step migration)
// Step 1: Mark as deprecated (deploy first, remove code references)
Schema::table('orders', function (Blueprint $table) {
    // Keep old column, add new column
    $table->renameColumn('shipping_method', 'deprecated_shipping_method');
});

// Step 2: Remove after code no longer references it (next deploy)
Schema::table('orders', function (Blueprint $table) {
    $table->dropColumn('deprecated_shipping_method');
});

// ❌ DANGEROUS: Change column type (locks table)
// Use two-step: add new column → migrate data → drop old column
Schema::table('orders', function (Blueprint $table) {
    $table->unsignedBigInteger('total_cents_new')->nullable()->after('total_cents');
});

// Data migration in a separate migration
DB::statement('UPDATE orders SET total_cents_new = total_cents WHERE total_cents_new IS NULL');

Schema::table('orders', function (Blueprint $table) {
    $table->dropColumn('total_cents');
    $table->renameColumn('total_cents_new', 'total_cents');
});

// Execution Successful

// routes/web.php
Route::get('/health', function () {
    $start = microtime(true);
    $checks = [];

    // Database check
    try {
        DB::connection()->getPdo();
        $checks['database'] = 'ok';
    } catch (\Throwable $e) {
        $checks['database'] = 'fail: ' . $e->getMessage();
    }

    // Redis check
    try {
        Cache::put('health_check', 'ok', 10);
        $checks['redis'] = Cache::get('health_check') === 'ok' ? 'ok' : 'fail';
    } catch (\Throwable $e) {
        $checks['redis'] = 'fail: ' . $e->getMessage();
    }

    // S3 check
    try {
        Storage::disk('s3')->put('health_check.txt', 'ok');
        $checks['storage'] = Storage::disk('s3')->get('health_check.txt') === 'ok' ? 'ok' : 'fail';
    } catch (\Throwable $e) {
        $checks['storage'] = 'fail: ' . $e->getMessage();
    }

    // Queue check
    try {
        $size = Queue::size('default');
        $checks['queue'] = $size < 10000 ? "ok (size: {$size})" : "warn (size: {$size})";
    } catch (\Throwable $e) {
        $checks['queue'] = 'fail: ' . $e->getMessage();
    }

    $latency = round((microtime(true) - $start) * 1000, 2);
    $allOk = collect($checks)->every(fn ($v) => str_starts_with($v, 'ok'));

    return response()->json([
        'status' => $allOk ? 'healthy' : 'unhealthy',
        'checks' => $checks,
        'latency_ms' => $latency,
        'version' => config('app.version'),
        'timestamp' => now()->toIso8601String(),
    ], $allOk ? 200 : 503);
});

// config/sentry.php
return [
    'dsn' => env('SENTRY_LARAVEL_DSN'),
    'traces_sample_rate' => env('SENTRY_TRACES_SAMPLE_RATE', 0.2),
    'send_default_pii' => false,
    'environment' => app()->environment(),
    'release' => config('app.version'),
];

// app/Exceptions/Handler.php
class Handler extends ExceptionHandler
{
    public function report(Throwable $e): void
    {
        if (app()->bound('sentry') && $this->shouldReport($e)) {
            \Sentry\configureScope(function (\Sentry\State\Scope $scope) {
                if ($user = auth()->user()) {
                    $scope->setUser([
                        'id' => $user->id,
                        'email' => $user->email,
                        'tenant_id' => $user->tenant_id,
                    ]);
                }

                $scope->setTag('app_version', config('app.version'));
                $scope->setExtra('request_url', request()->url());
                $scope->setExtra('request_method', request()->method());
            });
        }

        parent::report($e);
    }
}

// app/Providers/AppServiceProvider.php
class AppServiceProvider extends ServiceProvider
{
    public function boot(): void
    {
        if (app()->environment('local')) {
            Telescope::listenStorageNotifications();
        }

        Telescope::filter(function (IncomingEntry $entry) {
            if (app()->environment('local')) return true;
            return $entry->isReportableException() ||
                   $entry->isFailedJob() ||
                   $entry->isScheduledTask() ||
                   $entry->hasMonitoredTag();
        });
    }
}

// Execution Successful

#!/bin/bash
# backup-mysql.sh — Daily MySQL backup to S3
set -e

DB_NAME="shopmetrics"
S3_BUCKET="s3://shopmetrics-backups/mysql"
DATE=$(date +%Y%m%d_%H%M%S)
BACKUP_FILE="/tmp/shopmetrics_${DATE}.sql.gz"

echo "→ Starting backup: ${DATE}"

# Dump with consistent snapshot (no lock)
mysqldump \
  --host=mysql \
  --user=${DB_USERNAME} \
  --password=${DB_PASSWORD} \
  --single-transaction \
  --routines \
  --triggers \
  --quick \
  ${DB_NAME} | gzip > ${BACKUP_FILE}

# Upload to S3
aws s3 cp ${BACKUP_FILE} ${S3_BUCKET}/daily/shopmetrics_${DATE}.sql.gz

# Cleanup local file
rm -f ${BACKUP_FILE}

# Cleanup backups older than 30 days
aws s3 rm ${S3_BUCKET}/daily/ --recursive --exclude "*" --include "shopmetrics_*" \
  --query "Contents[?LastModified<='$(date -d '-30 days' +%Y-%m-%d)'].Key"

echo "✓ Backup complete: shopmetrics_${DATE}.sql.gz"

# Verify backup integrity
LATEST=$(aws s3 ls ${S3_BUCKET}/daily/ | tail -1 | awk '{print $4}')
aws s3 cp ${S3_BUCKET}/daily/${LATEST} /tmp/verify.sql.gz
if gzip -t /tmp/verify.sql.gz; then
    echo "✓ Backup integrity verified"
else
    echo "✗ Backup integrity check FAILED"
    exit 1
fi
rm -f /tmp/verify.sql.gz

// routes/console.php
use Illuminate\Support\Facades\Schedule;

// Daily database backup
Schedule::command('shopmetrics:backup-database')
    ->dailyAt('03:00')
    ->onOneServer()
    ->withoutOverlapping()
    ->emailOutputOnFailure('ops@shopmetrics.io');

// Hourly S3 storage sync (for report files)
Schedule::command('shopmetrics:sync-storage-backup')
    ->hourly()
    ->onOneServer();

// Weekly backup integrity test
Schedule::command('shopmetrics:verify-backup')
    ->weeklyOn(Schedule::SUNDAY, '04:00')
    ->onOneServer()
    ->emailOutputOnFailure('ops@shopmetrics.io');

// app/Console/Commands/BackupDatabase.php
class BackupDatabase extends Command
{
    protected $signature = 'shopmetrics:backup-database';
    protected $description = 'Backup MySQL database to S3';

    public function handle(): int
    {
        $filename = 'shopmetrics_' . now()->format('Ymd_His') . '.sql.gz';
        $tempPath = storage_path('app/backups/' . $filename);

        $this->info('Starting database backup...');

        $command = sprintf(
            'mysqldump --host=%s --user=%s --password=%s --single-transaction %s | gzip > %s',
            config('database.connections.mysql.host'),
            config('database.connections.mysql.username'),
            config('database.connections.mysql.password'),
            config('database.connections.mysql.database'),
            $tempPath
        );

        $exitCode = Process::run($command)->exitCode();

        if ($exitCode !== 0) {
            $this->error('Backup failed!');
            return self::FAILURE;
        }

        $s3Path = 'mysql/daily/' . $filename;
        Storage::disk('s3-backup')->put($s3Path, file_get_contents($tempPath));
        unlink($tempPath);

        $this->info("Backup uploaded: {$s3Path}");
        return self::SUCCESS;
    }
}

// Execution Successful

#!/bin/bash
# rollback.sh — Emergency rollback script
set -e

COMPOSE_FILE="docker-compose.prod.yml"
BACKUP_DIR="/opt/shopmetrics/backups"

echo "=== EMERGENCY ROLLBACK ==="

# Get previous image tag
PREVIOUS_TAG=$(cat ${BACKUP_DIR}/last_successful_deploy_tag.txt)
CURRENT_TAG=$(cat ${BACKUP_DIR}/current_deploy_tag.txt)

echo "Current: ${CURRENT_TAG}"
echo "Rolling back to: ${PREVIOUS_TAG}"

# Step 1: Set image tag to previous version
export IMAGE_TAG=${PREVIOUS_TAG}

# Step 2: Pull previous image
docker compose -f ${COMPOSE_FILE} pull app queue-high queue-default scheduler

# Step 3: Start previous version containers
docker compose -f ${COMPOSE_FILE} up -d --no-deps --scale app=2 app
docker compose -f ${COMPOSE_FILE} up -d --no-deps queue-high queue-default scheduler

# Step 4: Wait and verify
sleep 10
if curl -sf http://localhost:8080/health > /dev/null; then
    echo "✓ Health check passed after rollback"
else
    echo "✗ Health check FAILED after rollback — MANUAL INTERVENTION REQUIRED"
    exit 1
fi

# Step 5: Check if DB migration rollback is needed
echo "⚠ Check if database migrations need rollback:"
echo "  Run: docker compose -f ${COMPOSE_FILE} exec -T app php artisan migrate:status"
echo "  If needed: docker compose -f ${COMPOSE_FILE} exec -T app php artisan migrate:rollback --step=N"

# Step 6: Update current tag
echo "${PREVIOUS_TAG}" > ${BACKUP_DIR}/current_deploy_tag.txt

# Step 7: Notify team
echo "⚠ ROLLBACK COMPLETE — Notify team immediately"
echo "  Rolled back from ${CURRENT_TAG} to ${PREVIOUS_TAG}"

CONTAINER ID   IMAGE     STATUS    
abc123         latest    Up 2 hours

#!/bin/bash
# ============================================
# Comprehensive: ShopMetrics full deployment script
# Covers: backup → deploy → migrate → verify → rollback plan
# ============================================

set -euo pipefail

DEPLOY_TAG=${1:-$(git rev-parse --short HEAD)}
COMPOSE_FILE="docker-compose.prod.yml"
BACKUP_DIR="/opt/shopmetrics/backups"
SLACK_WEBHOOK="${SLACK_WEBHOOK_URL}"

log() { echo "[$(date +%H:%M:%S)] $1"; }
notify() { curl -s -X POST "${SLACK_WEBHOOK}" -H 'Content-type: application/json' --data "{\"text\":\"$1\"}" > /dev/null 2>&1 || true; }

# Pre-deploy: Backup
log "→ Step 1/10: Database backup"
php artisan shopmetrics:backup-database
log "✓ Backup complete"

# Pre-deploy: Save current state
log "→ Step 2/10: Saving current state"
cat ${BACKUP_DIR}/current_deploy_tag.txt > ${BACKUP_DIR}/last_successful_deploy_tag.txt 2>/dev/null || true
echo "${DEPLOY_TAG}" > ${BACKUP_DIR}/current_deploy_tag.txt
log "✓ State saved (rolling back to: $(cat ${BACKUP_DIR}/last_successful_deploy_tag.txt 2>/dev/null || echo 'N/A'))"

# Deploy: Pull and start
log "→ Step 3/10: Pulling image ${DEPLOY_TAG}"
export IMAGE_TAG=${DEPLOY_TAG}
docker compose -f ${COMPOSE_FILE} pull app queue-high queue-default scheduler

log "→ Step 4/10: Starting new containers"
docker compose -f ${COMPOSE_FILE} up -d --no-deps --remove-orphans app queue-high queue-default scheduler

# Health check with retry
log "→ Step 5/10: Health check"
for i in $(seq 1 6); do
    if curl -sf http://localhost:8080/health > /dev/null 2>&1; then
        log "✓ Health check passed"
        break
    fi
    if [ $i -eq 6 ]; then
        log "✗ Health check failed after 30s — ROLLING BACK"
        bash /opt/shopmetrics/rollback.sh
        notify "🚨 ShopMetrics deploy FAILED (health check) — rolled back"
        exit 1
    fi
    log "  Waiting... ($i/6)"
    sleep 5
done

# Migrate
log "→ Step 6/10: Running migrations"
docker compose -f ${COMPOSE_FILE} exec -T app php artisan migrate --force
if [ $? -ne 0 ]; then
    log "✗ Migration failed — ROLLING BACK"
    bash /opt/shopmetrics/rollback.sh
    notify "🚨 ShopMetrics deploy FAILED (migration) — rolled back"
    exit 1
fi
log "✓ Migrations complete"

# Optimize
log "→ Step 7/10: Caching"
docker compose -f ${COMPOSE_FILE} exec -T app php artisan optimize:clear
docker compose -f ${COMPOSE_FILE} exec -T app php artisan optimize
docker compose -f ${COMPOSE_FILE} exec -T app php artisan queue:restart
log "✓ Caches rebuilt"

# Post-deploy verification
log "→ Step 8/10: Smoke tests"
SMOKE_PASS=true
for endpoint in "/health" "/api/v1/plans"; do
    STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://shopmetrics.io${endpoint}")
    if [ "$STATUS" -ge 400 ]; then
        log "✗ Smoke test failed: ${endpoint} → HTTP ${STATUS}"
        SMOKE_PASS=false
    fi
done
if [ "$SMOKE_PASS" = false ]; then
    log "✗ Smoke tests failed — ROLLING BACK"
    bash /opt/shopmetrics/rollback.sh
    notify "🚨 ShopMetrics deploy FAILED (smoke test) — rolled back"
    exit 1
fi
log "✓ Smoke tests passed"

# Cleanup
log "→ Step 9/10: Cleanup"
docker image prune -f > /dev/null
log "✓ Old images cleaned"

# Success
log "→ Step 10/10: Final verification"
sleep 5
ERRORS=$(docker compose -f ${COMPOSE_FILE} logs --since 2m app 2>&1 | grep -c "ERROR" || true)
if [ "${ERRORS}" -gt 5 ]; then
    log "⚠ High error rate detected: ${ERRORS} errors in 2 minutes"
    notify "⚠️ ShopMetrics deployed but high error rate: ${ERRORS} errors"
else
    log "✓ Deployment successful: ${DEPLOY_TAG}"
    notify "✅ ShopMetrics deployed successfully: ${DEPLOY_TAG}"
fi

log "=== DEPLOY COMPLETE ==="