Jenkinsfile.api.prod

pipeline {
    agent any

    environment {
        GHCR_URL = 'ghcr.io/poolchaos'
        IMAGE_NAME = 'serelo-api'
        // DELTA-12: Production server from Jenkins credential (not hardcoded)
        PROD_SERVER = credentials('SERELO_PROD_SERVER')
        DOCKER_BUILDKIT = '1'
    }

    options {
        skipDefaultCheckout(true)
        disableConcurrentBuilds()
        timeout(time: 60, unit: 'MINUTES')
    }

    stages {
        stage('Initial Memory Check & Build Lock') {
            steps {
                script {
                    echo "🚦 === INITIAL MEMORY ASSESSMENT ==="

                    sh '''
                        echo "📊 Initial memory status:"
                        free -h

                        # Get available memory
                        AVAILABLE_MB=$(free -m | awk 'NR==2{printf "%d", $7}')
                        echo "💾 Available memory: ${AVAILABLE_MB}MB"

                        # Basic Docker cleanup
                        echo "🧹 Basic Docker cleanup..."
                        docker container prune -f 2>/dev/null || echo "Container cleanup completed"
                        docker image prune -f 2>/dev/null || echo "Image cleanup completed"

                        # Simple build lock
                        echo "🔒 Creating build lock..."
                        mkdir -p /tmp/jenkins-serelo-api-build
                        echo "$(date +%s):$$:serelo-api" > /tmp/jenkins-serelo-api-build/lock

                        # Memory assessment
                        AVAILABLE_MB=$(free -m | awk 'NR==2{printf "%d", $7}')
                        if [ "$AVAILABLE_MB" -lt "200" ]; then
                            echo "🚨 CRITICAL MEMORY: ${AVAILABLE_MB}MB - build may fail"
                            echo "💡 Proceeding but expect potential failures"
                        elif [ "$AVAILABLE_MB" -lt "400" ]; then
                            echo "⚠️ LOW MEMORY: ${AVAILABLE_MB}MB - using minimal strategy"
                        else
                            echo "✅ SUFFICIENT MEMORY: ${AVAILABLE_MB}MB"
                        fi

                        echo "✅ Ready for workspace setup"
                    '''
                }
            }
        }

        stage('Workspace Setup') {
            steps {
                echo "🧹 === WORKSPACE SETUP ==="
                script {
                    // Smart cleanup - preserve caches
                    sh '''
                        if [ -d ".git" ]; then
                            echo "🔄 Selective cleanup preserving caches..."
                            git clean -ffdx -e "**/__pycache__" -e "**/.pytest_cache"
                        else
                            echo "🆕 Fresh workspace detected"
                        fi
                    '''

                    // Explicit checkout
                    checkout([
                        $class: 'GitSCM',
                        branches: [[name: '*/main']],
                        doGenerateSubmoduleConfigurations: false,
                        extensions: [
                            [$class: 'CleanBeforeCheckout'],
                            [$class: 'CloneOption', depth: 1, shallow: true]
                        ],
                        userRemoteConfigs: [[
                            url: 'https://github.com/Poolchaos/serelo.git',
                            credentialsId: 'github-repo-access'
                        ]]
                    ])

                    // Set environment variables
                    env.IMAGE_TAG = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
                    env.FULL_IMAGE_NAME = "${GHCR_URL}/${IMAGE_NAME}:${env.IMAGE_TAG}"

                    // Automated verification
                    sh '''
                        echo "✅ Checkout completed"
                        echo "📋 Image: ${FULL_IMAGE_NAME}"

                        # Verify workspace structure
                        echo "🔍 Verifying workspace structure..."

                        # Check critical files exist
                        if [ ! -d "backend" ]; then
                            echo "❌ ERROR: backend directory not found!"
                            pwd && ls -la
                            exit 1
                        fi

                        if [ ! -f "backend/Dockerfile" ]; then
                            echo "❌ ERROR: backend/Dockerfile not found!"
                            exit 1
                        fi

                        if [ ! -f "backend/requirements.txt" ]; then
                            echo "❌ ERROR: backend/requirements.txt not found!"
                            exit 1
                        fi

                        echo "✅ Backend structure verified"
                        echo "✅ Moving to test stage"
                    '''
                }
            }
        }

        stage('Run Tests') {
            steps {
                script {
                    echo "🧪 === BACKEND TEST SUITE ==="
                    catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
                        sh '''
                            set -e
                            echo "📦 Running backend tests inside Python container..."
                            docker run --rm \
                                -v "$(pwd)/backend:/app" \
                                -w /app \
                                python:3.11-slim \
                                sh -c "pip install --quiet -r requirements.txt && python -m pytest tests/ --tb=short --no-header -q --timeout=120 --ignore=tests/integration"
                            echo "✅ All tests passed"
                        '''
                    }
                    // Remove test image immediately - do not leave it accumulating on disk
                    sh 'docker rmi python:3.11-slim 2>/dev/null || true'
                }
            }
        }

        stage('Docker Build') {
            steps {
                script {
                    echo "🐳 === DOCKER BUILD ==="

                    sh '''
                        set -e

                        echo "🔍 Pre-build verification..."
                        echo "Build context: $(pwd)/backend"
                        echo "Dockerfile location: $(ls -la backend/Dockerfile)"

                        # Pull latest image for cache
                        echo "📥 Pulling latest image for build cache..."
                        docker pull ${GHCR_URL}/${IMAGE_NAME}:latest || echo "No cache image available, will build from scratch"

                        # Memory-adaptive build strategy
                        AVAILABLE_MB=$(free -m | awk 'NR==2{printf "%d", $7}')
                        echo "💾 Available memory: ${AVAILABLE_MB}MB"

                        if [ "$AVAILABLE_MB" -gt "1200" ]; then
                            echo "💪 Using optimal build settings..."
                            MEMORY_LIMIT="1024m"
                            SWAP_LIMIT="1536m"
                            CPU_QUOTA="50000"
                        elif [ "$AVAILABLE_MB" -gt "800" ]; then
                            echo "⚖️ Using balanced build settings..."
                            MEMORY_LIMIT="800m"
                            SWAP_LIMIT="1200m"
                            CPU_QUOTA="30000"
                        else
                            echo "🐌 Using emergency minimal build settings..."
                            MEMORY_LIMIT="600m"
                            SWAP_LIMIT="800m"
                            CPU_QUOTA="20000"
                        fi

                        # Build API Docker image
                        echo "📦 Building Serelo API Docker image..."
                        echo "Memory: $MEMORY_LIMIT, Swap: $SWAP_LIMIT, CPU: $CPU_QUOTA"

                        docker build \\
                            --progress=plain \\
                            --cache-from ${GHCR_URL}/${IMAGE_NAME}:latest \\
                            --build-arg BUILDKIT_INLINE_CACHE=1 \\
                            --memory=$MEMORY_LIMIT \\
                            --memory-swap=$SWAP_LIMIT \\
                            --cpu-quota=$CPU_QUOTA \\
                            --cpu-period=100000 \\
                            --shm-size=64m \\
                            -f backend/Dockerfile \\
                            -t ${FULL_IMAGE_NAME} \\
                            backend/

                        echo "✅ Serelo API Docker image built successfully"
                    '''
                }
            }
        }

        stage('Container Security Scan') {
            steps {
                script {
                    echo "🔍 === CONTAINER VULNERABILITY SCAN ==="
                    catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
                        sh '''
                            set -e
                            if ! command -v trivy &> /dev/null; then
                                echo "📦 Installing Trivy..."
                                curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
                            fi
                            echo "🔍 Scanning ${FULL_IMAGE_NAME} for vulnerabilities..."
                            trivy image \
                                --severity HIGH,CRITICAL \
                                --exit-code 1 \
                                --ignore-unfixed \
                                --no-progress \
                                --cache-dir /tmp/trivy-cache \
                                ${FULL_IMAGE_NAME}
                            echo "✅ No HIGH/CRITICAL vulnerabilities found"
                        '''
                    }
                    // Clean Trivy cache immediately after scan
                    sh 'rm -rf /tmp/trivy-cache 2>/dev/null || true'
                }
            }
        }

        stage('Push to Registry') {
            steps {
                withCredentials([
                    string(credentialsId: 'GHCR_USERNAME', variable: 'GHCR_USER'),
                    string(credentialsId: 'GHCR_TOKEN', variable: 'GHCR_PAT')
                ]) {
                    script {
                        echo "📤 === API PUSH TO REGISTRY ==="

                        sh '''
                            set -e

                            echo "📤 Pushing Serelo API Docker image..."
                            echo "${GHCR_PAT}" | docker login ghcr.io -u "${GHCR_USER}" --password-stdin
                            docker push ${FULL_IMAGE_NAME}

                            echo "🏷️ Tagging and pushing as latest for cache..."
                            docker tag ${FULL_IMAGE_NAME} ${GHCR_URL}/${IMAGE_NAME}:latest
                            docker push ${GHCR_URL}/${IMAGE_NAME}:latest

                            echo "✅ API image pushed successfully: ${FULL_IMAGE_NAME}"
                            echo "✅ Cache image updated: ${GHCR_URL}/${IMAGE_NAME}:latest"
                        '''
                    }
                }
            }
        }

        stage('Approve Deploy') {
            steps {
                script {
                    echo "🤔 === DEPLOY CONFIRMATION REQUIRED ==="
                    echo "📋 Ready to deploy: ${env.FULL_IMAGE_NAME}"
                    echo "🎯 Target: Serelo API Production"
                    echo "⏰ Build will wait for deployment approval"

                    // Store current state
                    sh '''
                        echo "📸 Capturing current deployment state..."
                        ssh -o StrictHostKeyChecking=accept-new ${PROD_SERVER} "
                            echo '=== Current Serelo API Container Status ==='
                            docker ps --filter name=serelo-api || echo 'No API container running'
                            echo '=== Current Serelo API Images ==='
                            docker images | grep serelo-api | head -5 || echo 'No API images found'
                        " > deployment-state.log || echo "Could not capture remote state"

                        cat deployment-state.log || echo "No state log available"
                    '''

                    // Manual approval
                    def deploymentStrategy = input message: '🚀 Deploy Serelo API to Production?', ok: 'Deploy Now',
                          parameters: [
                              choice(name: 'DEPLOYMENT_STRATEGY',
                                     choices: ['rolling', 'immediate'],
                                     description: 'Choose deployment strategy')
                          ]

                    env.DEPLOYMENT_STRATEGY = deploymentStrategy

                    echo "✅ Deployment approved - proceeding with ${env.DEPLOYMENT_STRATEGY} strategy"
                }
            }
        }

        stage('Deploy Infrastructure') {
            steps {
                withCredentials([
                    string(credentialsId: 'SERELO_S3_ACCESS_KEY', variable: 'SERELO_S3_ACCESS_KEY'),
                    string(credentialsId: 'SERELO_S3_SECRET_KEY', variable: 'SERELO_S3_SECRET_KEY')
                ]) {
                    script {
                        echo "🏗️ === DEPLOYING INFRASTRUCTURE SERVICES ==="

                        sh '''
                            echo "🔗 Connecting to production server..."
                            ssh -o StrictHostKeyChecking=accept-new ${PROD_SERVER} "
                                set -e

                                # Ensure network exists
                                echo '🌐 Ensuring Docker network exists...'
                                docker network create serelo-net 2>/dev/null || echo 'Network already exists'

                                # Deploy Redis (only if not already running)
                                echo '🗄️ Checking Redis...'
                                if docker ps --filter name=serelo-redis --format '{{.Status}}' | grep -q 'Up'; then
                                    echo '✅ Redis already running - skipping restart'
                                else
                                    echo '🗄️ Starting Redis...'
                                    docker stop serelo-redis 2>/dev/null || true
                                    docker rm serelo-redis 2>/dev/null || true

                                    docker run -d --name serelo-redis \\
                                        --restart unless-stopped \\
                                        --network serelo-net \\
                                        -v serelo-redis-data:/data \\
                                        redis:7-alpine redis-server --appendonly yes
                                    echo '✅ Redis deployed'
                                fi

                                # Deploy MinIO (only if not already running)
                                echo '📦 Checking MinIO...'
                                if docker ps --filter name=serelo-minio --format '{{.Status}}' | grep -q 'Up'; then
                                    echo '✅ MinIO already running - skipping restart'
                                else
                                    echo '📦 Starting MinIO...'
                                    docker stop serelo-minio 2>/dev/null || true
                                    docker rm serelo-minio 2>/dev/null || true

                                    docker run -d --name serelo-minio \\
                                        --restart unless-stopped \\
                                        --network serelo-net \\
                                        -p 127.0.0.1:9000:9000 \\
                                        -p 127.0.0.1:9001:9001 \\
                                        -e MINIO_ROOT_USER='${SERELO_S3_ACCESS_KEY}' \
                                        -e MINIO_ROOT_PASSWORD='${SERELO_S3_SECRET_KEY}' \
                                        -v serelo-minio-data:/data \\
                                        minio/minio:RELEASE.2024-11-07T00-52-20Z server /data --console-address ':9001'
                                    echo '✅ MinIO deployed'
                                fi

                                # Wait for services to be ready
                                echo '⏳ Waiting for services to start...'
                                sleep 10

                                # Verify Redis
                                if docker exec serelo-redis redis-cli ping | grep -q PONG; then
                                    echo '✅ Redis is healthy'
                                else
                                    echo '⚠️ Redis health check failed'
                                fi

                                # Verify MinIO
                                if docker ps --filter name=serelo-minio --format '{{.Status}}' | grep -q 'Up'; then
                                    echo '✅ MinIO is running'
                                else
                                    echo '⚠️ MinIO health check failed'
                                fi

                                echo '✅ Infrastructure deployment completed'
                            "
                        '''

                        echo "✅ Infrastructure services deployed"
                    }
                }
            }
        }

        stage('Deploy to Production') {
            steps {
                withCredentials([
                    string(credentialsId: 'GHCR_USERNAME', variable: 'GHCR_USER'),
                    string(credentialsId: 'GHCR_TOKEN', variable: 'GHCR_PAT'),
                    string(credentialsId: 'SERELO_MONGODB_URL', variable: 'SERELO_MONGODB_URL'),
                    string(credentialsId: 'SERELO_MONGO_DB', variable: 'SERELO_MONGODB_DB_NAME'),
                    string(credentialsId: 'SERELO_REDIS_URL', variable: 'SERELO_REDIS_URL'),
                    string(credentialsId: 'SERELO_S3_ENDPOINT', variable: 'SERELO_S3_ENDPOINT'),
                    string(credentialsId: 'SERELO_S3_ACCESS_KEY', variable: 'SERELO_S3_ACCESS_KEY'),
                    string(credentialsId: 'SERELO_S3_SECRET_KEY', variable: 'SERELO_S3_SECRET_KEY'),
                    string(credentialsId: 'SERELO_JWT_SECRET', variable: 'SERELO_JWT_SECRET'),
                    string(credentialsId: 'SERELO_OPENAI_KEY', variable: 'SERELO_OPENAI_KEY'),
                    string(credentialsId: 'SERELO_PAYSTACK_SECRET', variable: 'SERELO_PAYSTACK_SECRET'),
                    string(credentialsId: 'SERELO_PAYSTACK_PUBLIC', variable: 'SERELO_PAYSTACK_PUBLIC'),
                    string(credentialsId: 'SERELO_SOCIAL_ENCRYPTION_KEY', variable: 'SERELO_SOCIAL_ENCRYPTION_KEY'),
                    string(credentialsId: 'SERELO_GOOGLE_AUTH_CLIENT_ID', variable: 'SERELO_GOOGLE_AUTH_CLIENT_ID'),
                    string(credentialsId: 'SERELO_GOOGLE_AUTH_CLIENT_SECRET', variable: 'SERELO_GOOGLE_AUTH_CLIENT_SECRET'),
                    string(credentialsId: 'SERELO_GITHUB_AUTH_CLIENT_ID', variable: 'SERELO_GITHUB_AUTH_CLIENT_ID'),
                    string(credentialsId: 'SERELO_GITHUB_AUTH_CLIENT_SECRET', variable: 'SERELO_GITHUB_AUTH_CLIENT_SECRET'),
                    string(credentialsId: 'SERELO_OAUTH_STATE_SECRET', variable: 'SERELO_OAUTH_STATE_SECRET'),
                    string(credentialsId: 'SERELO_FRONTEND_URL', variable: 'SERELO_FRONTEND_URL'),
                    string(credentialsId: 'SERELO_SOCIAL_ENCRYPTION_KEY', variable: 'SERELO_ENCRYPTION_KEY'),
                    string(credentialsId: 'SERELO_SMTP_HOST', variable: 'SERELO_SMTP_HOST'),
                    string(credentialsId: 'SERELO_SMTP_PORT', variable: 'SERELO_SMTP_PORT'),
                    string(credentialsId: 'SERELO_SMTP_USERNAME', variable: 'SERELO_SMTP_USERNAME'),
                    string(credentialsId: 'SERELO_SMTP_PASSWORD', variable: 'SERELO_SMTP_PASSWORD')
                ]) {
                    script {
                        echo "🚀 === DEPLOYING SERELO API TO PRODUCTION ==="

                        sh '''
                            echo "🔗 Connecting to production server..."
                            ssh -o StrictHostKeyChecking=accept-new ${PROD_SERVER} "
                                set -e
                                echo '🎯 Starting Serelo API deployment...'

                                # Login to registry
                                echo '🔐 Logging into container registry...'
                                echo '${GHCR_PAT}' | docker login ghcr.io -u '${GHCR_USER}' --password-stdin

                                # Pull new image
                                echo '📥 Pulling new Serelo API image...'
                                docker pull ${FULL_IMAGE_NAME}

                                # Stop and remove old container
                                echo '🔄 Stopping old container...'
                                docker stop serelo-api 2>/dev/null || echo 'No running container to stop'
                                docker rm serelo-api 2>/dev/null || echo 'No container to remove'

                                # Start new container
                                echo '🚀 Starting new Serelo API container...'
                                docker run -d --name serelo-api \\
                                    -p 127.0.0.1:8000:8000 \\
                                    --restart unless-stopped \\
                                    --network serelo-net \\
                                    -e MONGODB_URL='${SERELO_MONGODB_URL}' \
                                    -e MONGODB_DB_NAME='${SERELO_MONGODB_DB_NAME}' \
                                    -e REDIS_URL='${SERELO_REDIS_URL}' \
                                    -e S3_ENDPOINT='${SERELO_S3_ENDPOINT}' \\
                                    -e S3_ACCESS_KEY='${SERELO_S3_ACCESS_KEY}' \\
                                    -e S3_SECRET_KEY='${SERELO_S3_SECRET_KEY}' \\
                                    -e JWT_SECRET_KEY='${SERELO_JWT_SECRET}' \\
                                    -e JWT_ALGORITHM=HS256 \\
                                    -e ACCESS_TOKEN_EXPIRE_MINUTES=15 \\
                                    -e REFRESH_TOKEN_EXPIRE_DAYS=30 \\
                                    -e OPENAI_API_KEY='${SERELO_OPENAI_KEY}' \\
                                    -e PAYSTACK_SECRET_KEY='${SERELO_PAYSTACK_SECRET}' \\
                                    -e PAYSTACK_PUBLIC_KEY='${SERELO_PAYSTACK_PUBLIC}' \\
                                    -e SOCIAL_TOKEN_ENCRYPTION_KEY='${SERELO_SOCIAL_ENCRYPTION_KEY}' \\
                                    -e GOOGLE_AUTH_CLIENT_ID='${SERELO_GOOGLE_AUTH_CLIENT_ID}' \\
                                    -e GOOGLE_AUTH_CLIENT_SECRET='${SERELO_GOOGLE_AUTH_CLIENT_SECRET}' \\
                                    -e GITHUB_AUTH_CLIENT_ID='${SERELO_GITHUB_AUTH_CLIENT_ID}' \\
                                    -e GITHUB_AUTH_CLIENT_SECRET='${SERELO_GITHUB_AUTH_CLIENT_SECRET}' \\
                                    -e OAUTH_STATE_SECRET='${SERELO_OAUTH_STATE_SECRET}' \\
                                    -e FRONTEND_URL='${SERELO_FRONTEND_URL}' \\
                                    -e ENVIRONMENT=production \\
                                    -e ENCRYPTION_KEY='${SERELO_ENCRYPTION_KEY}' \\
                                    -e SMTP_HOST='${SERELO_SMTP_HOST}' \\
                                    -e SMTP_PORT='${SERELO_SMTP_PORT}' \\
                                    -e SMTP_USERNAME='${SERELO_SMTP_USERNAME}' \\
                                    -e SMTP_PASSWORD='${SERELO_SMTP_PASSWORD}' \\
                                    -e TEST_MODE=false \\
                                    ${FULL_IMAGE_NAME}

                                # Verify container is running
                                echo '✅ Verifying container...'
                                sleep 5
                                if docker ps --filter name=serelo-api --format '{{.Status}}' | grep -q 'Up'; then
                                    echo '✅ Container verified running'
                                else
                                    echo '❌ Container failed to start'
                                    docker logs serelo-api --tail 20 2>/dev/null || echo 'No logs available'
                                    exit 1
                                fi

                                echo '✅ Serelo API deployment completed'
                            "
                        '''

                        echo "✅ Serelo API deployed successfully to production"
                    }
                }
            }
        }

        stage('Seed Production Database') {
            steps {
                script {
                    echo "🌱 === SEEDING PRODUCTION DATABASE ==="

                    sh '''
                        ssh -o StrictHostKeyChecking=accept-new ${PROD_SERVER} "
                            echo '🌱 Running database seeding...'

                            # Wait for API to be fully ready
                            echo '⏳ Waiting for API to be ready...'
                            sleep 15

                            # Run seed_test_data.py in the API container
                            echo '📊 Seeding initial data (subscription plans, test users)...'
                            docker exec serelo-api python seed_test_data.py || echo '⚠️ Seeding completed with warnings (data may already exist)'

                            echo '✅ Database seeding completed'
                        "
                    '''

                    echo "✅ Production database seeding completed"
                }
            }
        }

        stage('Deploy Celery Worker') {
            steps {
                withCredentials([
                    string(credentialsId: 'SERELO_MONGODB_URL', variable: 'SERELO_MONGODB_URL'),
                    string(credentialsId: 'SERELO_MONGO_DB', variable: 'SERELO_MONGODB_DB_NAME'),
                    string(credentialsId: 'SERELO_REDIS_URL', variable: 'SERELO_REDIS_URL'),
                    string(credentialsId: 'SERELO_S3_ENDPOINT', variable: 'SERELO_S3_ENDPOINT'),
                    string(credentialsId: 'SERELO_S3_ACCESS_KEY', variable: 'SERELO_S3_ACCESS_KEY'),
                    string(credentialsId: 'SERELO_S3_SECRET_KEY', variable: 'SERELO_S3_SECRET_KEY'),
                    string(credentialsId: 'SERELO_JWT_SECRET', variable: 'SERELO_JWT_SECRET'),
                    string(credentialsId: 'SERELO_OPENAI_KEY', variable: 'SERELO_OPENAI_KEY'),
                    string(credentialsId: 'SERELO_SOCIAL_ENCRYPTION_KEY', variable: 'SERELO_SOCIAL_ENCRYPTION_KEY'),
                    string(credentialsId: 'SERELO_SOCIAL_ENCRYPTION_KEY', variable: 'SERELO_ENCRYPTION_KEY'),
                    string(credentialsId: 'SERELO_FRONTEND_URL', variable: 'SERELO_FRONTEND_URL'),
                    string(credentialsId: 'SERELO_SMTP_HOST', variable: 'SERELO_SMTP_HOST'),
                    string(credentialsId: 'SERELO_SMTP_PORT', variable: 'SERELO_SMTP_PORT'),
                    string(credentialsId: 'SERELO_SMTP_USERNAME', variable: 'SERELO_SMTP_USERNAME'),
                    string(credentialsId: 'SERELO_SMTP_PASSWORD', variable: 'SERELO_SMTP_PASSWORD')
                ]) {
                    script {
                        echo "🐝 === DEPLOYING CELERY WORKERS ==="

                        sh '''
                            ssh -o StrictHostKeyChecking=accept-new ${PROD_SERVER} "
                                set -e

                                # Deploy Celery Worker
                                echo '🐝 Deploying Celery Worker...'
                                docker stop serelo-celery-worker 2>/dev/null || echo 'No worker to stop'
                                docker rm serelo-celery-worker 2>/dev/null || echo 'No worker to remove'

                                docker run -d --name serelo-celery-worker \\
                                    --restart unless-stopped \\
                                    --network serelo-net \\
                                    -e MONGODB_URL='${SERELO_MONGODB_URL}' \
                                    -e MONGODB_DB_NAME='${SERELO_MONGODB_DB_NAME}' \
                                    -e REDIS_URL='${SERELO_REDIS_URL}' \
                                    -e S3_ENDPOINT='${SERELO_S3_ENDPOINT}' \
                                    -e S3_ACCESS_KEY='${SERELO_S3_ACCESS_KEY}' \
                                    -e S3_SECRET_KEY='${SERELO_S3_SECRET_KEY}' \
                                    -e JWT_SECRET_KEY='${SERELO_JWT_SECRET}' \
                                    -e OPENAI_API_KEY='${SERELO_OPENAI_KEY}' \
                                    -e SOCIAL_TOKEN_ENCRYPTION_KEY='${SERELO_SOCIAL_ENCRYPTION_KEY}' \
                                    -e ENCRYPTION_KEY='${SERELO_ENCRYPTION_KEY}' \
                                    -e ENVIRONMENT=production \
                                    -e FRONTEND_URL='${SERELO_FRONTEND_URL}' \
                                    -e SMTP_HOST='${SERELO_SMTP_HOST}' \
                                    -e SMTP_PORT='${SERELO_SMTP_PORT}' \
                                    -e SMTP_USERNAME='${SERELO_SMTP_USERNAME}' \
                                    -e SMTP_PASSWORD='${SERELO_SMTP_PASSWORD}' \
                                    ${FULL_IMAGE_NAME} \\
                                    celery -A app.core.celery_app worker --loglevel=info --queues=parsing,generation,maintenance

                                # Deploy Celery Beat
                                echo '⏰ Deploying Celery Beat...'
                                docker stop serelo-celery-beat 2>/dev/null || echo 'No beat to stop'
                                docker rm serelo-celery-beat 2>/dev/null || echo 'No beat to remove'

                                docker run -d --name serelo-celery-beat \\
                                    --restart unless-stopped \\
                                    --network serelo-net \\
                                    -e MONGODB_URL='${SERELO_MONGODB_URL}' \
                                    -e MONGODB_DB_NAME='${SERELO_MONGODB_DB_NAME}' \
                                    -e REDIS_URL='${SERELO_REDIS_URL}' \
                                    -e S3_ENDPOINT='${SERELO_S3_ENDPOINT}' \
                                    -e S3_ACCESS_KEY='${SERELO_S3_ACCESS_KEY}' \
                                    -e S3_SECRET_KEY='${SERELO_S3_SECRET_KEY}' \
                                    -e JWT_SECRET_KEY='${SERELO_JWT_SECRET}' \
                                    -e OPENAI_API_KEY='${SERELO_OPENAI_KEY}' \
                                    -e ENCRYPTION_KEY='${SERELO_ENCRYPTION_KEY}' \
                                    -e ENVIRONMENT=production \
                                    ${FULL_IMAGE_NAME} \\
                                    celery -A app.core.celery_app beat --loglevel=info

                                echo '✅ Celery workers deployed'
                            "
                        '''
                    }
                }
            }
        }

        stage('Post-Deploy Verification') {
            steps {
                script {
                    echo "🔍 === POST-DEPLOY VERIFICATION ==="

                    sh '''
                        echo "🔍 Verifying Serelo API deployment..."
                        ssh -o StrictHostKeyChecking=accept-new ${PROD_SERVER} "
                            echo '=== Container Status ==='
                            docker ps --filter name=serelo

                            echo '=== API Logs (last 20 lines) ==='
                            docker logs --tail 20 serelo-api

                            echo '=== Health Check ==='
                            sleep 5
                            curl -f http://localhost:8000/health || echo 'Health check failed'

                            echo '=== Worker Status ==='
                            docker logs --tail 10 serelo-celery-worker
                        "
                    '''

                    echo "✅ Post-deploy verification completed"
                }
            }
        }

        stage('Deploy Server Monitor') {
            steps {
                script {
                    echo "📊 === DEPLOYING SERVER RESOURCE MONITOR ==="

                    sh '''
                        echo "📊 Installing server monitor script..."
                        cat scripts/server-monitor.sh | ssh -o StrictHostKeyChecking=accept-new ${PROD_SERVER} "
                            cat > /tmp/serelo-monitor.sh
                            chmod +x /tmp/serelo-monitor.sh
                            sudo mv /tmp/serelo-monitor.sh /usr/local/bin/serelo-monitor.sh

                            # Set up cron jobs if not already present
                            if ! crontab -l 2>/dev/null | grep -q 'serelo-monitor'; then
                                echo '📅 Adding monitor cron jobs...'
                                (crontab -l 2>/dev/null; echo '# Serelo resource monitor - every 15 minutes'; echo '*/15 * * * * /usr/local/bin/serelo-monitor.sh >> /var/log/serelo-monitor.log 2>&1') | crontab -
                            else
                                echo '✅ Monitor cron already configured'
                            fi

                            if ! crontab -l 2>/dev/null | grep -q 'docker image prune'; then
                                echo '📅 Adding Docker cleanup cron jobs...'
                                (crontab -l 2>/dev/null; echo '# Docker image cleanup - daily 3am'; echo '0 3 * * * docker image prune -a --filter until=24h -f >> /var/log/serelo-docker-cleanup.log 2>&1'; echo '# Full Docker system prune - Sunday 2am'; echo '0 2 * * 0 docker system prune -f >> /var/log/serelo-docker-cleanup.log 2>&1') | crontab -
                            else
                                echo '✅ Docker cleanup cron already configured'
                            fi

                            sudo touch /var/log/serelo-monitor.log /var/log/serelo-docker-cleanup.log 2>/dev/null || true
                            sudo chmod 644 /var/log/serelo-monitor.log /var/log/serelo-docker-cleanup.log 2>/dev/null || true

                            echo '📋 Active cron jobs:'
                            crontab -l

                            echo '✅ Server monitor deployed'
                        "
                    '''

                    echo "✅ Server monitor deployment completed"
                }
            }
        }
    }

    post {
        always {
            script {
                echo "🧹 === POST-BUILD CLEANUP ==="

                // Release build lock
                sh '''
                    echo "🔓 Releasing build lock..."
                    rm -f /tmp/jenkins-serelo-api-build/lock 2>/dev/null || echo "Lock cleanup completed"
                '''

                // Docker cleanup - remove ALL build-time images to prevent accumulation
                sh '''
                    set +e
                    echo "🐳 Docker cleanup..."

                    # Remove built image - it's in GHCR, no need to keep locally
                    docker rmi ${FULL_IMAGE_NAME} 2>/dev/null || true
                    docker rmi ${GHCR_URL}/${IMAGE_NAME}:latest 2>/dev/null || true

                    # Belt-and-suspenders: remove test images if still present
                    docker rmi python:3.11-slim 2>/dev/null || true

                    # Clean Trivy cache if still present
                    rm -rf /tmp/trivy-cache 2>/dev/null || true

                    # Prune stopped containers, dangling images, unused build cache
                    docker container prune -f 2>/dev/null || true
                    docker image prune -a --filter "until=1h" -f 2>/dev/null || true
                    docker builder prune -f 2>/dev/null || true

                    echo "📊 Final status:"
                    free -h
                    docker system df
                '''
            }
        }
        success {
            script {
                echo "🎉 === BUILD SUCCESS ==="
                echo "✅ Serelo API deployed successfully"
                echo "📋 Image: ${env.FULL_IMAGE_NAME}"
            }
        }
        failure {
            script {
                echo "💥 === BUILD FAILED ==="
                echo "❌ Serelo API build or deployment failed"
                echo "🔍 Check logs for details"

                // DELTA-06: Automatic rollback on deployment failure
                echo "🔄 Attempting rollback to previous image..."
                sh '''
                    set +e
                    ssh -o StrictHostKeyChecking=accept-new ${PROD_SERVER} "
                        PREVIOUS_IMAGE=\$(docker images ${GHCR_URL}/${IMAGE_NAME} --format '{{.Tag}}' | sed -n '2p')
                        if [ -n \"\$PREVIOUS_IMAGE\" ] && [ \"\$PREVIOUS_IMAGE\" != '<none>' ]; then
                            echo \"Rolling back to ${GHCR_URL}/${IMAGE_NAME}:\$PREVIOUS_IMAGE\"
                            docker stop serelo-api 2>/dev/null || true
                            docker rm serelo-api 2>/dev/null || true
                            echo 'Starting previous version...'
                            # Note: container will start with previous env vars from Docker
                            docker tag ${GHCR_URL}/${IMAGE_NAME}:\$PREVIOUS_IMAGE ${GHCR_URL}/${IMAGE_NAME}:rollback
                            echo 'Rollback image tagged. Manual restart required with correct env vars.'
                            echo 'Run: docker run -d --name serelo-api ... ${GHCR_URL}/${IMAGE_NAME}:rollback'
                        else
                            echo 'No previous image available for rollback'
                        fi
                    " 2>/dev/null || echo "Rollback attempt completed (may have failed)"
                '''
            }
        }
    }
}