-
Notifications
You must be signed in to change notification settings - Fork 1
257 lines (232 loc) · 11.5 KB
/
Copy pathdeploy-dev.yml
File metadata and controls
257 lines (232 loc) · 11.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
name: Deploy dev
# Deploy the checkin app to the AWS dev environment on every merge to main.
# Gated on CI: this fires only after the "CI" workflow succeeds for a push to
# main, builds an ARM64 image, runs DB migrations, rolls out the ECS service,
# and posts the result back onto the originating PR.
#
# Auth is GitHub OIDC (no static AWS keys) — assumes the checkin-deploy-dev role
# defined in ~/projects/treehouse/aws/infra/modules/checkin/iam.tf.
#
# Infra preconditions (NOT done here — see modules/checkin):
# - Terraform applied: OIDC role, ECR repo, ECS cluster/service, both task defs.
# - Secret VALUES set in Secrets Manager (checkin-dev/database-url, ...).
# - Dev DB (checkin_dev) bootstrapped (init.sql) so migrations can connect.
on:
workflow_run:
workflows: ["CI"]
types: [completed]
branches: [main]
permissions:
id-token: write # assume the deploy role via OIDC
contents: read # checkout
packages: write # push the image to ghcr.io
pull-requests: write # comment on the originating PR
concurrency:
# Group PER COMMIT (not a single shared group): with one shared group, GitHub
# collapses queued runs to the latest when merges land back-to-back, so an
# intermediate commit's deploy gets cancelled before it runs — losing the signal
# for which change broke things. Per-commit, every merge's deploy runs to
# completion and its result is pinned to that commit (re-runs of the SAME commit
# still dedup). Trade-off: two near-simultaneous merges can deploy in parallel
# (last update-service wins); acceptable for dev, and worth it for the visibility.
group: checkin-deploy-dev-${{ github.event.workflow_run.head_sha }}
cancel-in-progress: false # never abandon a deploy mid-rollout
env:
AWS_REGION: us-east-2
IMAGE_REPO: ghcr.io/innovationtreehouse/checkin-dev
ECS_CLUSTER: checkin-dev
ECS_SERVICE: checkin-dev
APP_TASK_FAMILY: checkin-dev
MIGRATE_TASK_FAMILY: checkin-migrate-dev
DEPLOY_ROLE: arn:aws:iam::639595353568:role/checkin-deploy-dev
APP_URL: https://ops-dev.innovationtreehouse.org
jobs:
deploy:
name: Build, migrate, roll out
# Only deploy a green push to main — skip CI runs from PRs / merge_group.
if: >-
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.event == 'push'
runs-on: ubuntu-24.04-arm # native ARM64 — task defs require arm64 images
outputs:
image_tag: ${{ steps.vars.outputs.sha }}
steps:
- name: Resolve deployed commit
id: vars
run: |
SHA='${{ github.event.workflow_run.head_sha }}'
echo "sha=$SHA" >> "$GITHUB_OUTPUT"
echo "short_sha=${SHA:0:7}" >> "$GITHUB_OUTPUT"
- name: Checkout deployed commit
uses: actions/checkout@v5
with:
ref: ${{ steps.vars.outputs.sha }}
- name: Configure AWS credentials (OIDC)
uses: aws-actions/configure-aws-credentials@v6
with:
role-to-assume: ${{ env.DEPLOY_ROLE }}
aws-region: ${{ env.AWS_REGION }}
- name: Log in to ghcr.io
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push image
id: build
env:
IMAGE: ${{ env.IMAGE_REPO }}:${{ steps.vars.outputs.sha }}
run: |
set -euo pipefail
# Native arm64 runner -> a plain build produces the arm64 image the
# ECS task defs require (runtime_platform.cpu_architecture = ARM64).
docker build \
--build-arg NEXT_PUBLIC_GIT_SHA="${{ steps.vars.outputs.sha }}" \
-f checkin-app/Dockerfile \
-t "$IMAGE" .
docker push "$IMAGE"
echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
- name: Run database migrations
env:
IMAGE: ${{ steps.build.outputs.image }}
run: |
set -euo pipefail
# Register a new revision of the migrate task def pointing at the new
# image (run-task can't override the image, only command/env), keeping
# only the fields register-task-definition accepts. Goes through a real
# temp file: the runner's aws CLI cannot read file:///dev/stdin.
aws ecs describe-task-definition \
--task-definition "$MIGRATE_TASK_FAMILY" \
--query taskDefinition --output json \
| jq --arg IMG "$IMAGE" '
.containerDefinitions[0].image = $IMG
| {family, taskRoleArn, executionRoleArn, networkMode,
containerDefinitions, requiresCompatibilities, cpu, memory,
runtimePlatform}' > "$RUNNER_TEMP/migrate-taskdef.json"
MIGRATE_ARN=$(aws ecs register-task-definition \
--cli-input-json "file://$RUNNER_TEMP/migrate-taskdef.json" \
--query 'taskDefinition.taskDefinitionArn' --output text)
echo "Registered migrate task def: $MIGRATE_ARN"
# Reuse the service's own network config (subnets / SG / public IP) so
# the one-off task lands exactly where the service runs.
NETWORK_CONFIG=$(aws ecs describe-services \
--cluster "$ECS_CLUSTER" --services "$ECS_SERVICE" \
--query 'services[0].networkConfiguration' --output json)
# The shared Aurora cluster is Serverless v2 with min capacity 0 and
# auto-pause: when idle (dev DB: always), the first connection races
# the ~30s resume and Prisma fails fast with P1001. Override the
# container command with a retry loop so the task survives the wake-up.
cat > "$RUNNER_TEMP/migrate-overrides.json" <<'EOF'
{
"containerOverrides": [{
"name": "checkin-migrate",
"command": ["sh", "-c",
"cd checkin-app && for i in 1 2 3 4 5; do npx prisma migrate deploy && exit 0; echo \"attempt $i failed — DB may be resuming from auto-pause; retrying in 20s\"; sleep 20; done; exit 1"]
}]
}
EOF
# FARGATE_SPOT (associated on the cluster) instead of on-demand
# --launch-type FARGATE: matches prod's migrate path; a Spot reclaim
# mid-run fails the step loudly and a re-run is safe (one small
# migration per release, retry loop below handles DB wake-up).
TASK_ARN=$(aws ecs run-task \
--cluster "$ECS_CLUSTER" \
--task-definition "$MIGRATE_ARN" \
--capacity-provider-strategy capacityProvider=FARGATE_SPOT,weight=1 \
--network-configuration "$NETWORK_CONFIG" \
--overrides "file://$RUNNER_TEMP/migrate-overrides.json" \
--query 'tasks[0].taskArn' --output text)
echo "Migration task: $TASK_ARN"
aws ecs wait tasks-stopped --cluster "$ECS_CLUSTER" --tasks "$TASK_ARN"
EXIT_CODE=$(aws ecs describe-tasks \
--cluster "$ECS_CLUSTER" --tasks "$TASK_ARN" \
--query 'tasks[0].containers[0].exitCode' --output text)
echo "Migration exit code: $EXIT_CODE"
# Surface the container's own output either way — without this, a failure
# only shows the exit code here and the real Prisma error (P3009 etc.)
# hides in CloudWatch, which is how the 2026-07-02 P3009 wedge went
# undiagnosed for a day.
TASK_ID="${TASK_ARN##*/}"
echo "--- migrate container logs (/ecs/checkin-migrate-dev) ---"
aws logs get-log-events \
--log-group-name /ecs/checkin-migrate-dev \
--log-stream-name "ecs/checkin-migrate/$TASK_ID" \
--start-from-head --limit 200 \
--query 'events[].message' --output text | tr '\t' '\n' || echo "(could not fetch logs)"
echo "--- end migrate container logs ---"
if [ "$EXIT_CODE" != "0" ]; then
echo "::error::Database migration failed (exit $EXIT_CODE) — see migrate container logs above"
exit 1
fi
- name: Deploy to ECS
env:
IMAGE: ${{ steps.build.outputs.image }}
run: |
set -euo pipefail
# Register a new app task-def revision with the new image. The service
# has lifecycle.ignore_changes on task_definition, so CI owns rollout.
# Same temp-file dance as the migrate step (no /dev/stdin on the runner).
aws ecs describe-task-definition \
--task-definition "$APP_TASK_FAMILY" \
--query taskDefinition --output json \
| jq --arg IMG "$IMAGE" '
.containerDefinitions[0].image = $IMG
| {family, taskRoleArn, executionRoleArn, networkMode,
containerDefinitions, volumes, placementConstraints,
requiresCompatibilities, cpu, memory, runtimePlatform}' > "$RUNNER_TEMP/app-taskdef.json"
APP_ARN=$(aws ecs register-task-definition \
--cli-input-json "file://$RUNNER_TEMP/app-taskdef.json" \
--query 'taskDefinition.taskDefinitionArn' --output text)
echo "Registered app task def: $APP_ARN"
# --desired-count 1: with scale-to-zero (infra#107) dev normally sits at
# desired 0, where services-stable is vacuously true and a crash-looping
# image would deploy "green". Booting one task makes the wait real; the
# scaler tears it back down after the idle window.
aws ecs update-service \
--cluster "$ECS_CLUSTER" --service "$ECS_SERVICE" \
--task-definition "$APP_ARN" \
--desired-count 1 >/dev/null
echo "Waiting for the service to reach steady state..."
aws ecs wait services-stable --cluster "$ECS_CLUSTER" --services "$ECS_SERVICE"
echo "Service is stable."
- name: Report result on the PR
if: always()
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SHA: ${{ steps.vars.outputs.sha }}
SHORT_SHA: ${{ steps.vars.outputs.short_sha }}
STATUS: ${{ job.status }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
# Find the PR the merged commit came from. Empty for direct pushes.
PR=$(gh api "repos/${{ github.repository }}/commits/$SHA/pulls" \
--jq '.[0].number' 2>/dev/null || true)
if [ -z "$PR" ] || [ "$PR" = "null" ]; then
echo "No PR associated with $SHA — skipping comment."
exit 0
fi
if [ "$STATUS" = "success" ]; then
BODY="### ✅ Deployed to dev
Commit \`$SHORT_SHA\` is live on [$APP_URL]($APP_URL).
- Image: \`$IMAGE_REPO:$SHORT_SHA\`
- Migrations: applied · Service: stable
- [Deploy run]($RUN_URL)"
else
BODY="### ❌ Dev deploy failed
Commit \`$SHORT_SHA\` did **not** deploy to dev (status: \`$STATUS\`).
- [Deploy run]($RUN_URL) — check the logs.
- The dev environment still runs the previous image."
fi
gh pr comment "$PR" --body "$BODY"
echo "Posted result on PR #$PR"
# s-read rides every dev deploy (issue #235's decision: the checkin pipeline
# owns s-read deploys; deploy-s-read.yml's manual dispatch remains for ad-hoc
# runs). Runs after the app is out; same commit; migrate-and-code mode.
s-read:
name: Deploy s-read (dev)
needs: deploy
uses: ./.github/workflows/deploy-s-read.yml
with:
environment: dev
ref: ${{ github.event.workflow_run.head_sha }}