deploy.yml

name: Deploy

on:
  push:
    branches: [main]
  release:
    types: [published]
  workflow_dispatch:
    inputs:
      environment:
        description: 'Environment to deploy to'
        required: true
        default: 'staging'
        type: choice
        options:
          - staging
          - production

# Deploy jobs only check out code and SSH outward — no write scopes needed.
permissions:
  contents: read

env:
  PYTHON_VERSION: '3.11'
  NODE_VERSION: '20'

jobs:
  # ============================================
  # Run Tests First (Quality Gate)
  # ============================================
  test:
    name: Run Test Suite
    uses: ./.github/workflows/test.yml


  # ============================================
  # Build images (CI, never on the server)
  # ============================================
  # The whole point of #1121: an artifact that is PROVEN to build before it can
  # reach a host. The shape this replaces built on the box, so a failed npm ci
  # or uv sync left new code beside a dead process, with no previous image to
  # restart.
  #
  # Two jobs rather than one matrix because NEXT_PUBLIC_* are inlined into the
  # client bundle at build time — staging and production genuinely need
  # different images, and each job needs its own `environment:` to see its own
  # secrets (the #1112 lesson: an environment secret is empty in a job that
  # does not declare the environment).
  docker-build-staging:
    name: Build images (staging)
    runs-on: ubuntu-latest
    needs: test
    if: |
      (github.event_name == 'push') ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging')
    environment: staging
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Log in to GHCR
        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Buildx
        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1

      - name: Build and push backend
        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
        with:
          context: .
          push: true
          tags: |
            ghcr.io/frankbria/codeframe-backend:${{ github.sha }}
            ghcr.io/frankbria/codeframe-backend:staging
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Build and push frontend
        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
        with:
          context: ./web-ui
          push: true
          # Build args, not runtime env: Next inlines NEXT_PUBLIC_* into the
          # client bundle and snapshots BACKEND_ORIGIN into the standalone
          # server's config. Setting any of them on the container does nothing.
          build-args: |
            NEXT_PUBLIC_API_URL=${{ secrets.API_URL }}
            NEXT_PUBLIC_WS_URL=${{ secrets.WS_URL }}
            NEXT_PUBLIC_SSE_URL=${{ secrets.API_URL }}
            BACKEND_ORIGIN=http://backend:14200
          tags: |
            ghcr.io/frankbria/codeframe-frontend:${{ github.sha }}
            ghcr.io/frankbria/codeframe-frontend:staging
          cache-from: type=gha
          cache-to: type=gha,mode=max

  docker-build-production:
    name: Build images (production)
    runs-on: ubuntu-latest
    needs: test
    if: |
      (github.event_name == 'release') ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'production')
    environment: production
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Log in to GHCR
        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Buildx
        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1

      - name: Build and push backend
        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
        with:
          context: .
          push: true
          tags: |
            ghcr.io/frankbria/codeframe-backend:${{ github.sha }}
            ghcr.io/frankbria/codeframe-backend:production
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Build and push frontend
        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
        with:
          context: ./web-ui
          push: true
          build-args: |
            NEXT_PUBLIC_API_URL=${{ secrets.API_URL }}
            NEXT_PUBLIC_WS_URL=${{ secrets.WS_URL }}
            NEXT_PUBLIC_SSE_URL=${{ secrets.API_URL }}
            BACKEND_ORIGIN=http://backend:14200
          tags: |
            ghcr.io/frankbria/codeframe-frontend:${{ github.sha }}
            ghcr.io/frankbria/codeframe-frontend:production
          cache-from: type=gha
          cache-to: type=gha,mode=max

  # ============================================
  # Deploy to Staging
  # ============================================
  deploy-staging:
    name: Deploy to Staging
    runs-on: ubuntu-latest
    permissions:
      contents: read
      # The job pipes this token to the server for `docker login ghcr.io`.
      # Without it the images build fine and the deploy then fails at pull —
      # GHCR packages are private by default even for a public repo.
      packages: read
    # Gate: tests pass AND both images built. A deploy can no longer reach the
    # host unless the artifact it will run already exists.
    needs: [test, docker-build-staging]
    # Serialize staging deploys. The build-on-server shape made this essential
    # — overlapping runs raced `uv venv --clear` against a live process. Since
    # #1121 the deploy is only a pull and a `compose up`, so the hazard is much
    # smaller, but two runs still converge on the same containers and ports and
    # the later image should win cleanly. Queue rather than cancel: never
    # interrupt an in-flight deploy and leave the box half-updated.
    concurrency:
      group: deploy-staging
      cancel-in-progress: false
    if: |
      (github.event_name == 'push' && github.ref == 'refs/heads/main') ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging')
    environment:
      name: staging
      # dev.codeframeapp.com has DNS to this box but no nginx site and no
      # certificate, so the shared VPS serves another project's cert and TLS
      # fails. The GitHub deployment link therefore looked like staging was
      # down when it was serving fine. dev.codeframe.sh is the real host.
      url: https://dev.codeframe.sh

    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Set up SSH
        uses: webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555 # v0.10.0
        with:
          ssh-private-key: ${{ secrets.SSH_KEY }}

      # The VPS firewalls port 22 off the public internet; the runner reaches it
      # over Tailscale instead. HOST must be the box's tailnet MagicDNS name (or
      # 100.x address), and SSH_KNOWN_HOSTS must be labelled with that same HOST.
      # Box side: join the tailnet and `ufw allow in on tailscale0 to any port 22`.
      - name: Connect to Tailscale
        uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3
        with:
          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
          tags: tag:ci

      - name: Add server to known hosts
        # Pinned host key (no live ssh-keyscan TOFU). Populate the
        # SSH_KNOWN_HOSTS secret from a trusted session with:
        #   ssh-keyscan -H <host> (verify the fingerprint out-of-band first)
        env:
          SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
        run: |
          if [ -z "${SSH_KNOWN_HOSTS}" ]; then
            echo "❌ SSH_KNOWN_HOSTS secret is not set — the deploy cannot verify the server host key."
            echo "   Populate it from a trusted machine (verify the fingerprint out-of-band first):"
            echo "     ssh-keyscan -H <host> | gh secret set SSH_KNOWN_HOSTS --env <staging|production>"
            exit 1
          fi
          mkdir -p ~/.ssh
          printf '%s\n' "${SSH_KNOWN_HOSTS}" >> ~/.ssh/known_hosts

      - name: Create environment file
        env:
          REMOTE_HOST: ${{ secrets.HOST }}
          REMOTE_USER: ${{ secrets.USER }}
          REMOTE_PATH: ${{ secrets.PROJECT_PATH }}
          ENV_ANTHROPIC_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          ENV_OPENAI_KEY: ${{ secrets.OPENAI_API_KEY }}
          ENV_AUTH_SECRET: ${{ secrets.AUTH_SECRET }}
          ENV_DATABASE_PATH: ${{ secrets.DATABASE_PATH }}
          ENV_WORKSPACE_ROOT: ${{ secrets.WORKSPACE_ROOT }}
          ENV_API_HOST: ${{ secrets.API_HOST }}
          ENV_API_PORT: ${{ secrets.API_PORT }}
          ENV_CORS: ${{ secrets.CORS_ORIGINS }}
          ENV_API_URL: ${{ secrets.API_URL }}
          ENV_WS_URL: ${{ secrets.WS_URL }}
          ENV_LOG_LEVEL: ${{ secrets.LOG_LEVEL }}
          ENV_LOG_FILE: ${{ secrets.LOG_FILE }}
          ENV_ENVIRONMENT: ${{ secrets.ENVIRONMENT }}
          ENV_DEBUG: ${{ secrets.DEBUG }}
          ENV_HOT_RELOAD: ${{ secrets.HOT_RELOAD }}
        run: |
          echo "📝 Creating .env.staging file..."

          # Workspace allowlist (#896). The backend refuses to start when auth
          # is enabled and this is unset, because an empty allowlist lets any
          # authenticated user open a terminal session in ANY host directory.
          # Optional secret with a sane default, so the deploy doesn't start
          # failing on a secret that has never been set.
          WORKSPACE_ROOT="${ENV_WORKSPACE_ROOT:-${REMOTE_PATH}/workspaces}"

          # Fail fast if the AUTH_SECRET secret is missing/empty. The backend
          # hard-fails on the default secret when auth is enabled (issue #643),
          # which otherwise only surfaces as a health-check timeout much later.
          if [ -z "${ENV_AUTH_SECRET}" ] || ! printf '%s' "${ENV_AUTH_SECRET}" | grep -q '[^[:space:]]'; then
            echo "❌ AUTH_SECRET GitHub Actions secret is not set (or blank/whitespace-only)."
            echo "   Set it to a secure random value, e.g.:"
            echo "     gh secret set AUTH_SECRET --body \"\$(openssl rand -hex 32)\""
            exit 1
          fi

          # Build env file content safely using printf (no shell interpretation)
          ENV_CONTENT=$(printf '%s\n' \
            "# CodeFRAME Environment Configuration" \
            "# Auto-generated by GitHub Actions deployment" \
            "" \
            "# AI Provider API Keys" \
            "ANTHROPIC_API_KEY=${ENV_ANTHROPIC_KEY}" \
            "OPENAI_API_KEY=${ENV_OPENAI_KEY}" \
            "" \
            "# Authentication" \
            "# Required: the backend hard-fails on the default secret when auth" \
            "# is enabled (the default) — see issue #643." \
            "AUTH_SECRET=${ENV_AUTH_SECRET}" \
            "" \
            "# Database Configuration" \
            "DATABASE_PATH=${ENV_DATABASE_PATH}" \
            "" \
            "# Workspace allowlist (#896) — permitted roots for workspaces and" \
            "# interactive sessions. A session workspace_path becomes a terminal" \
            "# shell cwd, so an unset allowlist is a shell anywhere on the host;" \
            "# the backend refuses to start without it when auth is enabled." \
            "WORKSPACE_ROOT=${WORKSPACE_ROOT}" \
            "CODEFRAME_DEPLOYMENT_MODE=self_hosted" \
            "" \
            "# Status Server Configuration" \
            "API_HOST=${ENV_API_HOST}" \
            "API_PORT=${ENV_API_PORT}" \
            "CORS_ALLOWED_ORIGINS=${ENV_CORS}" \
            "" \
            "# Web UI Configuration" \
            "NEXT_PUBLIC_API_URL=${ENV_API_URL}" \
            "NEXT_PUBLIC_WS_URL=${ENV_WS_URL}" \
            "" \
            "# Logging Configuration" \
            "LOG_LEVEL=${ENV_LOG_LEVEL}" \
            "LOG_FILE=${ENV_LOG_FILE}" \
            "" \
            "# Environment & Development Flags" \
            "ENVIRONMENT=${ENV_ENVIRONMENT}" \
            "DEBUG=${ENV_DEBUG}" \
            "HOT_RELOAD=${ENV_HOT_RELOAD}" \
          )

          # Base64 encode to prevent any shell interpretation during transfer
          ENV_BASE64=$(echo "$ENV_CONTENT" | base64 -w 0)

          # Transfer and decode safely on remote, verify creation.
          # shellcheck disable=SC2029  # client-side expansion is the point:
          # ENV_BASE64 and REMOTE_PATH exist only on the runner, and the remote
          # side must receive their values, not their names.
          ssh "${REMOTE_USER}@${REMOTE_HOST}" "
            set -e
            echo '${ENV_BASE64}' | base64 -d > '${REMOTE_PATH}/.env.staging.tmp'
            if [ ! -s '${REMOTE_PATH}/.env.staging.tmp' ]; then
              echo '❌ Failed to create environment file (empty or missing)'
              rm -f '${REMOTE_PATH}/.env.staging.tmp'
              exit 1
            fi
            mv '${REMOTE_PATH}/.env.staging.tmp' '${REMOTE_PATH}/.env.staging'
            chmod 600 '${REMOTE_PATH}/.env.staging'
          "

          echo "✅ .env.staging created and verified"

      - name: Deploy to staging server
        env:
          # Short-lived: this token is valid only for the life of the job, so
          # the server needs no long-term registry credential and no new
          # secret has to be created for it.
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          # Legacy only — read once, to retire the PM2 processes that still
          # hold the ports. Nothing else in this workflow uses PM2 any more.
          LEGACY_BACKEND_NAME: ${{ secrets.PM2_BACKEND_NAME }}
          LEGACY_FRONTEND_NAME: ${{ secrets.PM2_FRONTEND_NAME }}
        run: |
          # The compose files are the only thing the host needs from the repo,
          # and they are small — copy them rather than keeping a git checkout
          # on the box just to read two YAML files.
          scp -q docker-compose.yml docker-compose.staging.yml \
            "${{ secrets.USER }}@${{ secrets.HOST }}:${{ secrets.PROJECT_PATH }}/"

          # shellcheck disable=SC2087  # client-side expansion is deliberate:
          # the image tag and registry token are runner-side values that must
          # be baked in. Forms that resolve on the server are escaped \${...}.
          # Keep this heredoc backtick-free — an unquoted delimiter substitutes
          # even '#' lines (#1130).
          ssh ${{ secrets.USER }}@${{ secrets.HOST }} "bash -s" << ENDSSH
            set -euo pipefail
            echo "🚀 Deploying staging (image ${{ github.sha }})..."

            cd ${{ secrets.PROJECT_PATH }}

            # No git pull, no uv sync, no npm ci, no build of any kind. The
            # images were built and proven in CI (#1121); this is a pull and a
            # restart, which is what makes it atomic and rollback-able.
            echo "${GHCR_TOKEN}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

            export IMAGE_TAG="${{ github.sha }}"
            export ENV_FILE=.env.staging
            # Same rule as production: the port the verify step curls is the
            # port the container publishes.
            export BACKEND_PORT="\$(grep -E '^API_PORT=' .env.staging 2>/dev/null | cut -d= -f2- || true)"
            export BACKEND_PORT="\${BACKEND_PORT:-14200}"
            COMPOSE="docker compose --env-file .env.staging -f docker-compose.yml -f docker-compose.staging.yml"

            # One-time migration off the PM2 layout. The old deploy kept the
            # SQLite DB and workspaces on the host filesystem; the containers
            # use named volumes, so without this the first containerised deploy
            # would come up with an empty database — silently losing the
            # operator's own login account. Idempotent: it only ever fires when
            # the volume has no DB yet, so re-running a deploy cannot clobber
            # live data with a stale copy.
            # One-time cutover: the PM2 processes still hold 14100/14200, so
            # the containers cannot bind and the deploy dies with
            # "address already in use". Stop them BY NAME — never a pm2 delete
            # of "all", which on this shared VPS takes out unrelated apps
            # (#912). No backticks in here: unquoted heredoc (#1130).
            # A no-op forever after, and on any host that never ran PM2.
            if command -v pm2 >/dev/null 2>&1; then
              for legacy in "${LEGACY_BACKEND_NAME}" "${LEGACY_FRONTEND_NAME}"; do
                if [ -n "\${legacy}" ] && pm2 describe "\${legacy}" >/dev/null 2>&1; then
                  echo "🛑 Retiring PM2 process \${legacy}"
                  pm2 delete "\${legacy}" >/dev/null 2>&1 || true
                fi
              done
              pm2 save >/dev/null 2>&1 || true
            fi

            LEGACY_DB="\$(grep -E '^DATABASE_PATH=' .env.staging 2>/dev/null | cut -d= -f2-)"
            if [ -n "\${LEGACY_DB}" ] && [ -f "\${LEGACY_DB}" ]; then
              echo "🗃️  Legacy DB at \${LEGACY_DB} — seeding the volume if it is empty"
              # Labelled so compose adopts it as its own. Without these it
              # warns "already exists but was not created by Docker Compose" on
              # every single deploy — noise that trains you to ignore the log.
              docker volume create \
                --label com.docker.compose.project=codeframe \
                --label com.docker.compose.volume=codeframe-data \
                codeframe_codeframe-data >/dev/null
              # The copy runs inside a throwaway container because a named
              # volume's host path is a docker implementation detail. SRC comes
              # in as env so no path has to survive another round of quoting.
              docker run --rm \
                -e SRC="/legacy/\$(basename "\${LEGACY_DB}")" \
                -v codeframe_codeframe-data:/data \
                -v "\$(dirname "\${LEGACY_DB}")":/legacy:ro \
                alpine:3.20 sh -c '
                  if [ -f /data/codeframe.db ]; then
                    echo "   volume already has a database — left alone"
                  else
                    cp "\$SRC" /data/codeframe.db && echo "   copied into the volume"
                  fi
                  # The backend runs as uid 10001 (non-root). Docker only
                  # applies the image directory'"'"'s ownership when it initialises
                  # an EMPTY named volume — this one was pre-populated above, so
                  # without this chown the DB lands root-owned and the app
                  # cannot write to it. Verified on the box.
                  chown -R 10001:10001 /data'
            fi

            echo "📥 Pulling images..."
            \$COMPOSE pull

            echo "🔁 Restarting services..."
            \$COMPOSE up -d --remove-orphans

            echo "⏳ Waiting for health..."
            \$COMPOSE ps

            docker logout ghcr.io >/dev/null 2>&1 || true
            echo "✅ Deployment to staging complete!"
          ENDSSH

      - name: Verify deployment
        run: |
          echo "🔍 Verifying staging deployment..."
          MAX_ATTEMPTS=12
          SLEEP_SECONDS=5

          for i in $(seq 1 "$MAX_ATTEMPTS"); do
            echo "Health check attempt $i/$MAX_ATTEMPTS..."
            if ssh ${{ secrets.USER }}@${{ secrets.HOST }} "curl -sf http://localhost:${{ secrets.API_PORT }}/health"; then
              echo "✅ Health check passed on attempt $i"
              exit 0
            fi
            if [ "$i" -lt "$MAX_ATTEMPTS" ]; then
              echo "⏳ Waiting ${SLEEP_SECONDS}s before retry..."
              sleep "$SLEEP_SECONDS"
            fi
          done

          echo "❌ Health check failed after $MAX_ATTEMPTS attempts"
          # Surface the cause: a silent `curl -sf` gave no reason. Same
          # diagnostics as before, translated from PM2 to compose (#1121) —
          # container state plus recent logs, so a crash-loop is diagnosable
          # from the run without SSHing in.
          # shellcheck disable=SC2029  # client-side expansion is the point.
          echo "::group::Backend diagnostics (container state + recent logs)"
          ssh ${{ secrets.USER }}@${{ secrets.HOST }} "
            cd '${{ secrets.PROJECT_PATH }}' || exit 0
            COMPOSE='docker compose -f docker-compose.yml -f docker-compose.staging.yml'
            \$COMPOSE ps || true
            echo '--- last 60 lines of backend logs ---'
            \$COMPOSE logs --tail 60 backend 2>&1 || echo '(no backend container)'
            echo '--- last 30 lines of frontend logs ---'
            \$COMPOSE logs --tail 30 frontend 2>&1 || echo '(no frontend container)'
          " || true
          echo "::endgroup::"
          exit 1

      - name: Deployment summary
        env:
          # Branch names carry the same metacharacter risk as tags (#933).
          BRANCH_NAME: ${{ github.ref_name }}
        run: |
          {
            echo "## Staging Deployment Summary"
            echo ""
            echo "- **Branch**: $BRANCH_NAME"
            echo "- **Commit**: \`${{ github.sha }}\`"
            echo "- **Deployed by**: ${{ github.actor }}"
            echo "- **Time**: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          } >> "$GITHUB_STEP_SUMMARY"

  # ============================================
  # Deploy to Production
  # ============================================
  deploy-production:
    name: Deploy to Production
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: read
    # Gate: tests pass AND the images exist. See deploy-staging.
    needs: [test, docker-build-production]
    # Serialize production deploys (own group, independent of staging).
    concurrency:
      group: deploy-production
      cancel-in-progress: false
    if: |
      (github.event_name == 'release') ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'production')
    environment:
      name: production
      # No url: production has never been deployed and has no host. A
      # placeholder (codeframe.example.com) shipped here for a long time,
      # which reads on the GitHub deployments page as a real target (#1143).

    steps:
      # The `production` environment holds ZERO secrets while this job
      # references two dozen, and GitHub resolves a missing environment secret
      # to an empty string with no warning. Today that surfaces as an SSH auth
      # failure five steps in — a symptom that says nothing about the cause.
      # Worse, populating only the SSH secrets would let the job proceed and
      # write a remote .env with AUTH_SECRET= — an empty JWT signing key.
      #
      # So: fail first, name every missing secret, and say what to do. This is
      # option 3 from #1143 (guard, don't delete): staging is the only deploy
      # target today, but launch is the direction of travel, so removing the
      # job would just have to be rewritten later.
      #
      # Values are read from env, never interpolated into the script — an
      # expression inside a run block executes on the runner (see the release
      # job below for the same rule).
      - name: Preflight — production must actually be configured
        env:
          # Required: without any of these the deploy is broken or unsafe.
          SSH_KEY: ${{ secrets.SSH_KEY }}
          SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
          TS_OAUTH_CLIENT_ID: ${{ secrets.TS_OAUTH_CLIENT_ID }}
          TS_OAUTH_SECRET: ${{ secrets.TS_OAUTH_SECRET }}
          HOST: ${{ secrets.HOST }}
          USER: ${{ secrets.USER }}
          PROJECT_PATH: ${{ secrets.PROJECT_PATH }}
          AUTH_SECRET: ${{ secrets.AUTH_SECRET }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          API_PORT: ${{ secrets.API_PORT }}
          API_URL: ${{ secrets.API_URL }}
          WS_URL: ${{ secrets.WS_URL }}
          CORS_ORIGINS: ${{ secrets.CORS_ORIGINS }}
        run: |
          set -euo pipefail
          REQUIRED="SSH_KEY SSH_KNOWN_HOSTS TS_OAUTH_CLIENT_ID TS_OAUTH_SECRET
                    HOST USER PROJECT_PATH AUTH_SECRET ANTHROPIC_API_KEY
                    API_PORT API_URL WS_URL CORS_ORIGINS"

          missing=""
          for name in $REQUIRED; do
            # Whitespace-only counts as missing: a secret set to " " is not set.
            value="${!name:-}"
            if [ -z "$(printf '%s' "$value" | tr -d '[:space:]')" ]; then
              missing="${missing} ${name}"
            fi
          done

          if [ -n "$missing" ]; then
            echo "::error::The production environment is not configured."
            echo ""
            echo "Missing (or blank) secrets:"
            for name in $missing; do echo "  - ${name}"; done
            echo ""
            echo "GitHub exposes environment secrets only to jobs declaring that"
            echo "environment, and 'production' currently holds none. Staging is"
            echo "the only working deploy target. Set them with:"
            echo "  gh secret set <NAME> --env production"
            echo "and see the staging environment for the full expected set."
            exit 1
          fi

          echo "✅ All required production secrets are present and non-empty."

      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Set up SSH
        uses: webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555 # v0.10.0
        with:
          ssh-private-key: ${{ secrets.SSH_KEY }}

      # The VPS firewalls port 22 off the public internet; the runner reaches it
      # over Tailscale instead. HOST must be the box's tailnet MagicDNS name (or
      # 100.x address), and SSH_KNOWN_HOSTS must be labelled with that same HOST.
      # Box side: join the tailnet and `ufw allow in on tailscale0 to any port 22`.
      - name: Connect to Tailscale
        uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3
        with:
          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
          tags: tag:ci

      - name: Add server to known hosts
        # Pinned host key (no live ssh-keyscan TOFU). Populate the
        # SSH_KNOWN_HOSTS secret from a trusted session with:
        #   ssh-keyscan -H <host> (verify the fingerprint out-of-band first)
        env:
          SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
        run: |
          if [ -z "${SSH_KNOWN_HOSTS}" ]; then
            echo "❌ SSH_KNOWN_HOSTS secret is not set — the deploy cannot verify the server host key."
            echo "   Populate it from a trusted machine (verify the fingerprint out-of-band first):"
            echo "     ssh-keyscan -H <host> | gh secret set SSH_KNOWN_HOSTS --env <staging|production>"
            exit 1
          fi
          mkdir -p ~/.ssh
          printf '%s\n' "${SSH_KNOWN_HOSTS}" >> ~/.ssh/known_hosts

      - name: Create environment file
        env:
          REMOTE_HOST: ${{ secrets.HOST }}
          REMOTE_USER: ${{ secrets.USER }}
          REMOTE_PATH: ${{ secrets.PROJECT_PATH }}
          ENV_ANTHROPIC_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          ENV_OPENAI_KEY: ${{ secrets.OPENAI_API_KEY }}
          ENV_AUTH_SECRET: ${{ secrets.AUTH_SECRET }}
          ENV_DATABASE_PATH: ${{ secrets.DATABASE_PATH }}
          ENV_WORKSPACE_ROOT: ${{ secrets.WORKSPACE_ROOT }}
          ENV_API_HOST: ${{ secrets.API_HOST }}
          ENV_API_PORT: ${{ secrets.API_PORT }}
          ENV_CORS: ${{ secrets.CORS_ORIGINS }}
          ENV_API_URL: ${{ secrets.API_URL }}
          ENV_WS_URL: ${{ secrets.WS_URL }}
          ENV_LOG_LEVEL: ${{ secrets.LOG_LEVEL }}
          ENV_LOG_FILE: ${{ secrets.LOG_FILE }}
          ENV_ENVIRONMENT: ${{ secrets.ENVIRONMENT }}
          ENV_DEBUG: ${{ secrets.DEBUG }}
          ENV_HOT_RELOAD: ${{ secrets.HOT_RELOAD }}
        run: |
          echo "📝 Creating .env.production file..."

          # Workspace allowlist (#896). The backend refuses to start when auth
          # is enabled and this is unset, because an empty allowlist lets any
          # authenticated user open a terminal session in ANY host directory.
          # Optional secret with a sane default, so the deploy doesn't start
          # failing on a secret that has never been set.
          WORKSPACE_ROOT="${ENV_WORKSPACE_ROOT:-${REMOTE_PATH}/workspaces}"

          # Fail fast if the AUTH_SECRET secret is missing/empty. The backend
          # hard-fails on the default secret when auth is enabled (issue #643),
          # which otherwise only surfaces as a health-check timeout much later.
          if [ -z "${ENV_AUTH_SECRET}" ] || ! printf '%s' "${ENV_AUTH_SECRET}" | grep -q '[^[:space:]]'; then
            echo "❌ AUTH_SECRET GitHub Actions secret is not set (or blank/whitespace-only)."
            echo "   Set it to a secure random value, e.g.:"
            echo "     gh secret set AUTH_SECRET --body \"\$(openssl rand -hex 32)\""
            exit 1
          fi

          # Build env file content safely using printf (no shell interpretation)
          ENV_CONTENT=$(printf '%s\n' \
            "# CodeFRAME Environment Configuration" \
            "# Auto-generated by GitHub Actions deployment" \
            "" \
            "# AI Provider API Keys" \
            "ANTHROPIC_API_KEY=${ENV_ANTHROPIC_KEY}" \
            "OPENAI_API_KEY=${ENV_OPENAI_KEY}" \
            "" \
            "# Authentication" \
            "# Required: the backend hard-fails on the default secret when auth" \
            "# is enabled (the default) — see issue #643." \
            "AUTH_SECRET=${ENV_AUTH_SECRET}" \
            "" \
            "# Database Configuration" \
            "DATABASE_PATH=${ENV_DATABASE_PATH}" \
            "" \
            "# Workspace allowlist (#896) — permitted roots for workspaces and" \
            "# interactive sessions. A session workspace_path becomes a terminal" \
            "# shell cwd, so an unset allowlist is a shell anywhere on the host;" \
            "# the backend refuses to start without it when auth is enabled." \
            "WORKSPACE_ROOT=${WORKSPACE_ROOT}" \
            "CODEFRAME_DEPLOYMENT_MODE=self_hosted" \
            "" \
            "# Status Server Configuration" \
            "API_HOST=${ENV_API_HOST}" \
            "API_PORT=${ENV_API_PORT}" \
            "CORS_ALLOWED_ORIGINS=${ENV_CORS}" \
            "" \
            "# Web UI Configuration" \
            "NEXT_PUBLIC_API_URL=${ENV_API_URL}" \
            "NEXT_PUBLIC_WS_URL=${ENV_WS_URL}" \
            "" \
            "# Logging Configuration" \
            "LOG_LEVEL=${ENV_LOG_LEVEL}" \
            "LOG_FILE=${ENV_LOG_FILE}" \
            "" \
            "# Environment & Development Flags" \
            "ENVIRONMENT=${ENV_ENVIRONMENT}" \
            "DEBUG=${ENV_DEBUG}" \
            "HOT_RELOAD=${ENV_HOT_RELOAD}" \
          )

          # Base64 encode to prevent any shell interpretation during transfer
          ENV_BASE64=$(echo "$ENV_CONTENT" | base64 -w 0)

          # Transfer and decode safely on remote, verify creation.
          # shellcheck disable=SC2029  # client-side expansion is the point:
          # ENV_BASE64 and REMOTE_PATH exist only on the runner, and the remote
          # side must receive their values, not their names.
          ssh "${REMOTE_USER}@${REMOTE_HOST}" "
            set -e
            echo '${ENV_BASE64}' | base64 -d > '${REMOTE_PATH}/.env.production.tmp'
            if [ ! -s '${REMOTE_PATH}/.env.production.tmp' ]; then
              echo '❌ Failed to create environment file (empty or missing)'
              rm -f '${REMOTE_PATH}/.env.production.tmp'
              exit 1
            fi
            mv '${REMOTE_PATH}/.env.production.tmp' '${REMOTE_PATH}/.env.production'
            chmod 600 '${REMOTE_PATH}/.env.production'
          "

          echo "✅ .env.production created and verified"

      - name: Create pre-deployment backup
        run: |
          # shellcheck disable=SC2087  # client-side expansion is deliberate:
          # ${NEXT_PUBLIC_*} and ${BACKEND_NAME}/${FRONTEND_NAME} are runner env
          # and must be baked in here. The forms that must resolve on the server
          # are escaped as \${...} below. Keep this heredoc backtick-free — an
          # unquoted delimiter substitutes even '#' lines (#1130).
          ssh ${{ secrets.USER }}@${{ secrets.HOST }} "bash -s" << ENDSSH
            set -e
            echo "💾 Creating pre-deployment backup..."

            PROJECT_PATH="${{ secrets.PROJECT_PATH }}"
            BACKUP_BASE="\${PROJECT_PATH}/backups"
            TIMESTAMP="\$(date +%Y%m%d-%H%M%S)"
            BACKUP_NAME="backup-\${TIMESTAMP}"
            TMP_BACKUP="/tmp/\${BACKUP_NAME}"
            FINAL_ARCHIVE="\${BACKUP_BASE}/\${BACKUP_NAME}.tar.gz"
            RETENTION_COUNT=10

            cd \${PROJECT_PATH}

            # Create persistent backup directory with proper permissions
            mkdir -p \${BACKUP_BASE}
            chmod 700 \${BACKUP_BASE}

            # Create temporary staging directory
            mkdir -p \${TMP_BACKUP}

            # Record current commit
            # Since #1121 the host has no checkout — the deploy pulls an image
            # instead — so these can legitimately find no repository. Under
            # set -e that aborted the whole backup step on a fresh host. The
            # commit is recorded when it happens to be knowable, and the image
            # tag (which IS the version now) is recorded always.
            {
              git rev-parse HEAD 2>/dev/null || echo "no-checkout"
              git log -1 --format="%H %s" 2>/dev/null || true
            } > \${TMP_BACKUP}/previous_commit.txt
            docker compose -f docker-compose.yml -f docker-compose.production.yml \
              images --format json > \${TMP_BACKUP}/previous_images.json 2>/dev/null || true
            echo "📝 Previous version: \$(head -1 \${TMP_BACKUP}/previous_commit.txt)"

            # Backup database
            if [ -f .codeframe/state.db ]; then
              cp .codeframe/state.db \${TMP_BACKUP}/
              echo "✅ Database backed up"
            fi

            # Backup config files
            if [ -d .codeframe ]; then
              cp -r .codeframe/config.* \${TMP_BACKUP}/ 2>/dev/null || true
              cp -r .codeframe/*.json \${TMP_BACKUP}/ 2>/dev/null || true
            fi

            # Backup environment files
            cp .env* \${TMP_BACKUP}/ 2>/dev/null || true
            cp docker-compose*.yml \${TMP_BACKUP}/ 2>/dev/null || true

            # Backup recent logs (last 1000 lines each to keep size manageable)
            if [ -d logs ]; then
              mkdir -p \${TMP_BACKUP}/logs
              for logfile in logs/*.log; do
                if [ -f "\$logfile" ]; then
                  tail -1000 "\$logfile" > "\${TMP_BACKUP}/logs/\$(basename \$logfile)" 2>/dev/null || true
                fi
              done
              echo "✅ Logs backed up"
            fi

            # Create compressed archive atomically
            echo "📦 Creating compressed archive..."
            tar -czf "\${TMP_BACKUP}.tar.gz" -C /tmp "\${BACKUP_NAME}" || {
              echo "❌ Failed to create backup archive"
              rm -rf \${TMP_BACKUP}
              exit 1
            }

            # Move to final location atomically
            mv "\${TMP_BACKUP}.tar.gz" "\${FINAL_ARCHIVE}" || {
              echo "❌ Failed to move backup to final location"
              rm -rf \${TMP_BACKUP} "\${TMP_BACKUP}.tar.gz"
              exit 1
            }

            # Set proper permissions on archive
            chmod 600 "\${FINAL_ARCHIVE}"

            # Cleanup temp directory
            rm -rf \${TMP_BACKUP}

            # Retention policy: keep last N backups
            echo "🗑️  Applying retention policy (keeping last \${RETENTION_COUNT} backups)..."
            cd \${BACKUP_BASE}
            ls -t backup-*.tar.gz 2>/dev/null | tail -n +\$((RETENTION_COUNT + 1)) | xargs -r rm -f

            # Report backup status
            BACKUP_SIZE=\$(du -h "\${FINAL_ARCHIVE}" | cut -f1)
            BACKUP_COUNT=\$(ls -1 backup-*.tar.gz 2>/dev/null | wc -l)
            echo "✅ Backup created: \${FINAL_ARCHIVE} (\${BACKUP_SIZE})"
            echo "📊 Total backups retained: \${BACKUP_COUNT}"
          ENDSSH

      - name: Deploy to production server
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          # Via env, never an expression inside the run script: an expression is
          # substituted as raw source text into the shell, an env var is data the
          # shell never re-parses (#933). Release tags are attacker-influenced —
          # git ref names may legally contain shell metacharacters.
          RELEASE_TAG: ${{ github.event.release.tag_name }}
        run: |
          set -euo pipefail

          # The tag is carried to the server base64-encoded so no metacharacter
          # in it is ever parsed by a shell on either side (#933).
          RELEASE_TAG_B64="$(printf '%s' "$RELEASE_TAG" | base64 | tr -d '\n')"

          scp -q docker-compose.yml docker-compose.production.yml \
            "${{ secrets.USER }}@${{ secrets.HOST }}:${{ secrets.PROJECT_PATH }}/"

          # shellcheck disable=SC2087  # client-side expansion is deliberate:
          # the image tag, encoded release tag and registry token are runner
          # values that must be baked in. Server-side forms are escaped
          # \${...}. Keep this heredoc backtick-free (#1130).
          ssh ${{ secrets.USER }}@${{ secrets.HOST }} "bash -s" << ENDSSH
            set -euo pipefail
            TAG_NAME="\$(printf '%s' '${RELEASE_TAG_B64}' | base64 -d)"
            echo "🚀 Deploying production release \${TAG_NAME} (image ${{ github.sha }})..."

            cd ${{ secrets.PROJECT_PATH }}

            # No git checkout of the tag, no build. The image was built from
            # this exact commit in CI and is addressed by its SHA (#1121).
            echo "${GHCR_TOKEN}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

            export IMAGE_TAG="${{ github.sha }}"
            export ENV_FILE=.env.production
            # One source of truth for the port: the same API_PORT the verify
            # step curls also decides what the container publishes.
            export BACKEND_PORT="\$(grep -E '^API_PORT=' .env.production 2>/dev/null | cut -d= -f2- || true)"
            export BACKEND_PORT="\${BACKEND_PORT:-14400}"
            COMPOSE="docker compose --env-file .env.production -f docker-compose.yml -f docker-compose.production.yml"

            echo "📥 Pulling images..."
            \$COMPOSE pull

            echo "🔁 Restarting services..."
            \$COMPOSE up -d --remove-orphans
            \$COMPOSE ps

            docker logout ghcr.io >/dev/null 2>&1 || true
            echo "✅ Deployment to production complete!"
          ENDSSH

      - name: Verify deployment
        run: |
          echo "🔍 Verifying production deployment..."
          MAX_ATTEMPTS=12
          SLEEP_SECONDS=5

          for i in $(seq 1 "$MAX_ATTEMPTS"); do
            echo "Health check attempt $i/$MAX_ATTEMPTS..."
            if ssh ${{ secrets.USER }}@${{ secrets.HOST }} "curl -sf http://localhost:${{ secrets.API_PORT }}/health"; then
              echo "✅ Health check passed on attempt $i"
              exit 0
            fi
            if [ "$i" -lt "$MAX_ATTEMPTS" ]; then
              echo "⏳ Waiting ${SLEEP_SECONDS}s before retry..."
              sleep "$SLEEP_SECONDS"
            fi
          done

          echo "❌ Health check failed after $MAX_ATTEMPTS attempts"
          # Surface the cause: a silent `curl -sf` gave no reason. Same
          # diagnostics as before, translated from PM2 to compose (#1121) —
          # container state plus recent logs, so a crash-loop is diagnosable
          # from the run without SSHing in.
          # shellcheck disable=SC2029  # client-side expansion is the point.
          echo "::group::Backend diagnostics (container state + recent logs)"
          ssh ${{ secrets.USER }}@${{ secrets.HOST }} "
            cd '${{ secrets.PROJECT_PATH }}' || exit 0
            COMPOSE='docker compose -f docker-compose.yml -f docker-compose.production.yml'
            \$COMPOSE ps || true
            echo '--- last 60 lines of backend logs ---'
            \$COMPOSE logs --tail 60 backend 2>&1 || echo '(no backend container)'
            echo '--- last 30 lines of frontend logs ---'
            \$COMPOSE logs --tail 30 frontend 2>&1 || echo '(no frontend container)'
          " || true
          echo "::endgroup::"
          exit 1

      - name: Deployment summary
        env:
          # Same class as the deploy step (#933): interpolating a ref name into
          # the run script executes `$( )` inside a tag on the runner. Not named
          # in the issue, but the identical defect one step later.
          RELEASE_VERSION: ${{ github.event.release.tag_name || github.ref_name }}
        run: |
          {
            echo "## Production Deployment Summary"
            echo ""
            echo "- **Version**: $RELEASE_VERSION"
            echo "- **Commit**: \`${{ github.sha }}\`"
            echo "- **Deployed by**: ${{ github.actor }}"
            echo "- **Time**: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          } >> "$GITHUB_STEP_SUMMARY"