Deploy #3006
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy | |
| on: | |
| workflow_run: | |
| workflows: [Build Check] | |
| types: [completed] | |
| branches: [main] | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: fly-deploy | |
| cancel-in-progress: false | |
| jobs: | |
| # Preflight: cheap sanity checks that must pass before we spend the | |
| # Fly build/deploy budget. If main has a broken state (e.g. two PRs | |
| # merged minutes apart reserved the same migration number), catch it | |
| # here instead of shipping a container that crashloops on boot. | |
| preflight: | |
| name: Preflight | |
| # workflow_run has access to deployment secrets, so only trust a successful | |
| # push run from this repository's main branch. Never deploy a PR run. | |
| if: >- | |
| github.event.workflow_run.conclusion == 'success' && | |
| github.event.workflow_run.event == 'push' && | |
| github.event.workflow_run.head_branch == 'main' && | |
| github.event.workflow_run.head_repository.full_name == github.repository | |
| runs-on: ubuntu-latest | |
| outputs: | |
| deploy_current: ${{ steps.current-main.outputs.deploy_current }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| fetch-depth: 0 | |
| - name: Require the tested SHA to still be current main | |
| id: current-main | |
| env: | |
| TESTED_SHA: ${{ github.event.workflow_run.head_sha }} | |
| run: | | |
| set -euo pipefail | |
| current_main_sha=$(git ls-remote origin refs/heads/main | awk '{print $1}') | |
| if [ -z "${current_main_sha}" ]; then | |
| echo "::error::Could not resolve the current origin/main SHA." | |
| exit 1 | |
| fi | |
| if [ "${TESTED_SHA}" != "${current_main_sha}" ]; then | |
| echo "::notice::Skipping stale tested SHA ${TESTED_SHA}; origin/main is ${current_main_sha}." | |
| echo "deploy_current=false" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "deploy_current=true" >> "${GITHUB_OUTPUT}" | |
| fi | |
| - name: No duplicate migration numbers | |
| if: steps.current-main.outputs.deploy_current == 'true' | |
| run: | | |
| dupes=$(ls server/src/db/migrations/*.sql \ | |
| | xargs -n1 basename \ | |
| | sed 's/_.*//' \ | |
| | sort \ | |
| | uniq -d) | |
| if [ -n "$dupes" ]; then | |
| echo "::error::Duplicate migration number prefixes on main: $dupes" | |
| echo "" | |
| echo "Deploy blocked — main would crashloop on boot. Land a renumber" | |
| echo "hotfix before the next push can deploy. Duplicate files:" | |
| for prefix in $dupes; do | |
| ls server/src/db/migrations/${prefix}_*.sql | |
| done | |
| exit 1 | |
| fi | |
| echo "No duplicate migration numbers on main." | |
| deploy: | |
| name: Deploy to Fly.io | |
| needs: preflight | |
| if: needs.preflight.outputs.deploy_current == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - name: Recheck tested SHA is still current main | |
| env: | |
| TESTED_SHA: ${{ github.event.workflow_run.head_sha }} | |
| run: | | |
| set -euo pipefail | |
| current_main_sha=$(git ls-remote origin refs/heads/main | awk '{print $1}') | |
| if [ -z "${current_main_sha}" ] || [ "${TESTED_SHA}" != "${current_main_sha}" ]; then | |
| echo "::error::Refusing to deploy stale SHA ${TESTED_SHA}; origin/main is ${current_main_sha:-unavailable}." | |
| exit 1 | |
| fi | |
| - name: Setup Fly CLI | |
| uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # 1.6 | |
| - name: Reject publisher crawl queue secret override | |
| env: | |
| FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
| run: | | |
| set -eo pipefail | |
| secrets=$(flyctl secrets list --json) | |
| override_count=$(echo "$secrets" | jq -er ' | |
| if type == "array" then | |
| [.[] | select((.Name // .name) == "PUBLISHER_CRAWL_QUEUE_ENABLED")] | length | |
| else | |
| error("expected Fly secrets JSON array") | |
| end') | |
| if [ "$override_count" -gt 0 ]; then | |
| echo "::error::PUBLISHER_CRAWL_QUEUE_ENABLED is set as a Fly secret and would override fly.toml. Remove the secret before deploying." | |
| exit 1 | |
| fi | |
| # flyctl deploy can fail for two very different reasons: | |
| # 1. The new image is broken (real failure — must block). | |
| # 2. Fly's machines API was unreachable during health-check polling | |
| # (transient — the app may already be healthy on the new image). | |
| # v2760 on 2026-05-19 was case 2: app served 200 throughout while | |
| # flyctl timed out polling the third machine because the machines API | |
| # was rate-limited from the earlier crashloop storm. The post-deploy | |
| # gates never ran. To distinguish, on failure we hit /health directly | |
| # — the same signal end users see — and only hard-fail if the app | |
| # itself is unreachable. The convergence gate below always remains | |
| # mandatory: a public health response cannot prove every machine | |
| # received the intended image and config. | |
| - name: Deploy | |
| id: deploy | |
| env: | |
| FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
| run: | | |
| set +e | |
| flyctl deploy --remote-only --wait-timeout 300 | |
| deploy_exit=$? | |
| set -e | |
| if [ $deploy_exit -eq 0 ]; then | |
| echo "deploy_outcome=success" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "::warning::flyctl deploy exited ${deploy_exit} — probing /health to distinguish API timeout from real failure" | |
| # Probe the Fly hostname directly, not the Cloudflare-fronted | |
| # public hostname. Cloudflare can cache /health responses and | |
| # a stale 200 would mask a real outage. The Fly hostname bypasses | |
| # CDN and reflects the actual app state on the machine. | |
| # Retry a few times in case the rolling restart hasn't settled — | |
| # a healthy app reaches /health=200 within a couple of seconds | |
| # once at least one machine is reachable. | |
| health_code="000" | |
| for attempt in 1 2 3 4; do | |
| sleep 5 | |
| health_code=$(curl -sS -o /tmp/health.json -w '%{http_code}' --max-time 15 https://adcp-docs.fly.dev/health || echo "000") | |
| echo "attempt ${attempt}: adcp-docs.fly.dev/health -> ${health_code}" | |
| if [ "${health_code}" = "200" ]; then break; fi | |
| done | |
| if [ -s /tmp/health.json ]; then | |
| echo "Body (first 500 chars):" | |
| head -c 500 /tmp/health.json | |
| echo | |
| fi | |
| if [ "${health_code}" = "200" ]; then | |
| echo "::warning::App is reachable on the Fly hostname (/health=200). Treating as fallback-success — the verification gate ran into a Fly API issue, not an app issue." | |
| echo "deploy_outcome=fallback-success" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "::error::flyctl deploy failed AND adcp-docs.fly.dev/health=${health_code} — real failure." | |
| echo "deploy_outcome=hard-fail" >> "$GITHUB_OUTPUT" | |
| exit 1 | |
| - name: Verify all app machines are on same image | |
| env: | |
| FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
| run: | | |
| # pipefail so a silent flyctl failure (empty $images, count=0) | |
| # surfaces as a real failure instead of being misread as | |
| # "no running web/worker machines." | |
| set -eo pipefail | |
| echo "Waiting for all app machines to converge on the same image..." | |
| # Verify the deployed value matches the checked-in kill-switch value. | |
| # This keeps the gate useful for both queue enablement and an | |
| # intentional emergency disablement. | |
| expected_queue=$(sed -nE 's/^[[:space:]]*PUBLISHER_CRAWL_QUEUE_ENABLED[[:space:]]*=[[:space:]]*"(true|false)"[[:space:]]*$/\1/p' fly.toml) | |
| if [ "$expected_queue" != "true" ] && [ "$expected_queue" != "false" ]; then | |
| echo "::error::fly.toml must define PUBLISHER_CRAWL_QUEUE_ENABLED exactly once as \"true\" or \"false\"" | |
| exit 1 | |
| fi | |
| # `flyctl deploy` can return while an immediate replacement is still | |
| # transitioning through `replacing`. Require every web/worker machine | |
| # to be started and on one digest, but give Fly a bounded settle window. | |
| for attempt in $(seq 1 12); do | |
| if ! machines=$(flyctl machines list --json); then | |
| echo "attempt ${attempt}/12: Fly Machines API request failed" | |
| elif ! summary=$(echo "$machines" | jq -ce --arg expected_queue "$expected_queue" ' | |
| if type != "array" then error("expected Fly machines JSON array") else . end | | |
| [.[] | select(.config.metadata.fly_process_group == "web" or .config.metadata.fly_process_group == "worker")] | | |
| { | |
| total: length, | |
| started: ([.[] | select(.state == "started")] | length), | |
| web: ([.[] | select(.config.metadata.fly_process_group == "web")] | length), | |
| worker: ([.[] | select(.config.metadata.fly_process_group == "worker")] | length), | |
| queue_matching: ([.[] | select(.config.env.PUBLISHER_CRAWL_QUEUE_ENABLED == $expected_queue)] | length), | |
| images: ([.[] | select(.state == "started") | .image_ref.digest] | unique) | |
| }'); then | |
| echo "attempt ${attempt}/12: invalid Fly Machines API response" | |
| else | |
| total=$(echo "$summary" | jq -r '.total') | |
| started=$(echo "$summary" | jq -r '.started') | |
| web=$(echo "$summary" | jq -r '.web') | |
| worker=$(echo "$summary" | jq -r '.worker') | |
| queue_matching=$(echo "$summary" | jq -r '.queue_matching') | |
| image_count=$(echo "$summary" | jq -r '.images | length') | |
| echo "attempt ${attempt}/12: total=${total} started=${started} web=${web} worker=${worker} queue_expected=${expected_queue} queue_matching=${queue_matching} image_count=${image_count}" | |
| if [ "$web" -ge 2 ] && [ "$worker" -ge 1 ] && \ | |
| [ "$started" -eq "$total" ] && [ "$queue_matching" -eq "$total" ] && \ | |
| [ "$image_count" -eq 1 ]; then | |
| echo "✅ All ${started} web/worker machines are started on one image with crawl queue=${expected_queue}" | |
| exit 0 | |
| fi | |
| fi | |
| if [ "$attempt" -lt 12 ]; then sleep 15; fi | |
| done | |
| echo "❌ App machines did not converge within 180 seconds" | |
| flyctl machines list | |
| exit 1 | |
| # Smoke each training-agent tenant URL after the rolling restart. | |
| # Catches production-only failures that don't surface in CI: SDK | |
| # init guards that throw under NODE_ENV=production (PR #3869), | |
| # in-memory task registry refusal (PR #3854), missing migrations, | |
| # etc. Both failure modes return 404 ("Tenant not registered") or | |
| # 5xx — both happen *before* auth, so an unauthenticated request | |
| # is sufficient: a healthy MCP endpoint returns 200 (when the | |
| # public token is set) or 401 (when it isn't). 4xx other than 401 | |
| # and any 5xx are deploy-fatal. | |
| - name: Smoke training-agent tenants | |
| env: | |
| PUBLIC_TEST_AGENT_TOKEN: ${{ secrets.PUBLIC_TEST_AGENT_TOKEN }} | |
| run: | | |
| set -uo pipefail | |
| base="https://test-agent.adcontextprotocol.org" | |
| paths=( | |
| "/sales/mcp" | |
| "/signals/mcp" | |
| "/governance/mcp" | |
| "/creative/mcp" | |
| "/creative-builder/mcp" | |
| "/brand/mcp" | |
| "/mcp" | |
| ) | |
| # Build curl args once. Omit the Authorization header when the secret | |
| # is unset so a strict-bearer-format server can't 400 us into a false | |
| # deploy failure — we accept 401/403 as healthy anyway. (SDK v5 used | |
| # 401 for invalid bearer; v6 uses 403. Both mean the registry resolved | |
| # and the MCP route is alive, which is what this smoke is checking.) | |
| auth_args=() | |
| if [ -n "${PUBLIC_TEST_AGENT_TOKEN:-}" ]; then | |
| auth_args=(-H "Authorization: Bearer ${PUBLIC_TEST_AGENT_TOKEN}") | |
| fi | |
| # Single attempt against one URL. Echoes the HTTP code; body in /tmp/smoke-body. | |
| probe() { | |
| local url="$1" | |
| curl -sS -o /tmp/smoke-body -w '%{http_code}' \ | |
| -X POST \ | |
| "${auth_args[@]}" \ | |
| -H "Content-Type: application/json" \ | |
| -H "Accept: application/json, text/event-stream" \ | |
| --max-time 15 \ | |
| -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' \ | |
| "${url}" || echo "000" | |
| } | |
| # Retry once on transient failure. Both bug modes we care about (#3854, | |
| # #3869) are deterministic — a healthy tenant won't blip on retry, but | |
| # a Cloudflare/Fly edge wobble during rolling-restart settle might. | |
| fail=0 | |
| for path in "${paths[@]}"; do | |
| url="${base}${path}" | |
| code=$(probe "${url}") | |
| case "${code}" in | |
| 200|401|403) | |
| echo "✅ ${path} → ${code}" | |
| continue | |
| ;; | |
| esac | |
| echo "⏳ ${path} → ${code}, retrying in 8s…" | |
| sleep 8 | |
| code=$(probe "${url}") | |
| case "${code}" in | |
| 200|401|403) | |
| echo "✅ ${path} → ${code} (after retry)" | |
| ;; | |
| *) | |
| echo "::error::Tenant smoke failed: ${path} returned HTTP ${code} (twice)" | |
| echo "Body (first 500 chars):" | |
| head -c 500 /tmp/smoke-body || true | |
| echo "" | |
| fail=1 | |
| ;; | |
| esac | |
| done | |
| if [ "${fail}" -ne 0 ]; then | |
| echo "::error::One or more training-agent tenants failed post-deploy smoke. Recent failures of this kind: PR #3854 (in-memory task registry refused under NODE_ENV=production), PR #3869 (noopJwksValidator threw under NODE_ENV=production, marking tenants disabled). Inspect Fly logs: flyctl logs --app <app>." | |
| exit 1 | |
| fi | |
| - name: Clean up stale console machines | |
| env: | |
| FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
| run: | | |
| # Destroy any leftover fly_app_console machines | |
| consoles=$(flyctl machines list --json | \ | |
| jq -r '.[] | select(.config.metadata.fly_process_group == "fly_app_console") | .id') | |
| if [ -n "$consoles" ]; then | |
| echo "Cleaning up stale console machines:" | |
| for id in $consoles; do | |
| echo " Destroying $id" | |
| flyctl machines destroy "$id" --force || true | |
| done | |
| else | |
| echo "No stale console machines found" | |
| fi | |
| - name: Setup Node.js for artifact build | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 | |
| with: | |
| node-version: "24" | |
| cache: "npm" | |
| - name: Install dependencies for artifact build | |
| env: | |
| PUPPETEER_SKIP_DOWNLOAD: "true" | |
| run: npm ci | |
| - name: Publish latest artifacts to R2 | |
| env: | |
| ADCP_ARTIFACT_R2_BUCKET: ${{ vars.ADCP_ARTIFACT_R2_BUCKET || 'adcp-artifacts' }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID || secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${R2_ACCOUNT_ID:-}" ] || [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then | |
| echo "::error::Missing R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, or R2_SECRET_ACCESS_KEY secret." | |
| exit 1 | |
| fi | |
| npm run backfill:cdn-artifacts -- --bucket "${ADCP_ARTIFACT_R2_BUCKET}" --build-latest --quiet | |
| # On any failure (hard or fallback-with-degraded-gates), snapshot Fly | |
| # state so the next-turn investigator doesn't have to fetch it | |
| # manually. v2760 on 2026-05-19 cost an extra debugging cycle because | |
| # the log was already past the retention horizon by the time we | |
| # opened it. Cheap to run, free if not needed. | |
| - name: Capture Fly forensics on failure | |
| if: failure() | |
| env: | |
| FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
| run: | | |
| mkdir -p /tmp/forensics | |
| flyctl status -a adcp-docs > /tmp/forensics/fly-status.txt 2>&1 || true | |
| flyctl machines list -a adcp-docs --json > /tmp/forensics/fly-machines.json 2>&1 || true | |
| flyctl releases -a adcp-docs > /tmp/forensics/fly-releases.txt 2>&1 || true | |
| # Boot logs have historically contained JWT fragments, idempotency | |
| # keys, and SDK init lines that echo env-derived config. The | |
| # artifact is repo-readable for 30 days, so redact obvious secret | |
| # shapes before persisting. This is best-effort — anything truly | |
| # secret-shaped that doesn't match these patterns will still slip | |
| # through, but most known leakage shapes are covered. | |
| flyctl logs -a adcp-docs --no-tail 2>&1 \ | |
| | tail -500 \ | |
| | grep -v -iE 'authorization|bearer |authn=|secret|password|api[-_]?key|sk-[a-z0-9]|wos_[a-z0-9]|fly_[a-z0-9]{20,}|eyJ[A-Za-z0-9_-]{10,}\.eyJ' \ | |
| > /tmp/forensics/fly-logs-tail.txt || true | |
| # Capture the deploy-outcome we computed so the artifact is | |
| # self-describing in 30 days. | |
| echo "deploy_outcome=${{ steps.deploy.outputs.deploy_outcome }}" > /tmp/forensics/summary.txt | |
| echo "github_sha=${{ github.event.workflow_run.head_sha }}" >> /tmp/forensics/summary.txt | |
| echo "github_run_id=${{ github.run_id }}" >> /tmp/forensics/summary.txt | |
| echo "note=fly-logs-tail.txt filtered for common secret shapes (Bearer/sk-/wos_/JWT/etc). Not exhaustive — treat as read-only." >> /tmp/forensics/summary.txt | |
| ls -la /tmp/forensics/ | |
| - name: Upload Fly forensics | |
| if: failure() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: fly-forensics-${{ github.run_id }} | |
| path: /tmp/forensics/ | |
| retention-days: 30 |